diff --git a/c/dbcdf.c b/c/dbcdf.c index f241606..efb990e 100644 --- a/c/dbcdf.c +++ b/c/dbcdf.c @@ -72,7 +72,7 @@ latency_add(struct vctr * const vctr, const u64 dt) // (parallel) load; nr <= nr_kvs static void -kvmap_batch_set_par(const struct forker_worker_info * const info, +kvmap_batch_put_par(const struct forker_worker_info * const info, const struct priv * const priv, const u64 nr) { const struct kvmap_api * const api = info->passdata[0]; @@ -95,7 +95,7 @@ kvmap_batch_set_par(const struct forker_worker_info * const info, } static void -kvmap_batch_set(const struct forker_worker_info * const info, +kvmap_batch_put(const struct forker_worker_info * const info, const struct priv * const priv, const u64 nr) { const struct kvmap_api * const api = info->passdata[0]; @@ -229,8 +229,8 @@ kvmap_worker(void * const ptr) switch (op) { case 'p': batch_func = kvmap_batch_pro; break; case 'g': batch_func = kvmap_batch_get; break; - case 's': batch_func = kvmap_batch_set; break; - case 'S': batch_func = kvmap_batch_set_par; break; + case 's': batch_func = kvmap_batch_put; break; + case 'S': batch_func = kvmap_batch_put_par; break; case 'd': batch_func = kvmap_batch_del; break; case 'n': batch_func = kvmap_batch_seek_next; break; case 'k': batch_func = kvmap_batch_seek_skip; break; diff --git a/c/dbtest1.c b/c/dbtest1.c new file mode 100644 index 0000000..d177c04 --- /dev/null +++ b/c/dbtest1.c @@ -0,0 +1,408 @@ +/* + * Copyright (c) 2016--2021 Wu, Xingbo + * + * All rights reserved. No warranty, explicit or implicit, provided. + */ +#define _GNU_SOURCE + +#include "lib.h" +#include "kv.h" + +struct priv { + void * ref; + u32 klen; + union { + u32 vlen; + u32 nscan; + }; + struct kv * tmp; + struct kv * out; +}; + +#define XSA ((0)) // Set-All +#define XSS ((1)) // Set-Success +#define XDA ((2)) +#define XDS ((3)) +#define XGA ((4)) +#define XGS ((5)) +#define XPA ((6)) +#define XPS ((7)) +#define XNA ((8)) +#define XNS ((9)) +#define XKA ((10)) +#define XKS ((11)) +#define VCTRSZ ((12)) + + static bool +kvmap_analyze(void * const passdata[2], const u64 dt, const struct vctr * const va, struct damp * const d, char * const out) +{ + (void)passdata; + size_t v[VCTRSZ]; + for (u64 i = 0; i < VCTRSZ; i++) + v[i] = vctr_get(va, i); + + const u64 nrop = v[XSA] + v[XDA] + v[XGA] + v[XPA] + v[XNA] + v[XKA]; + const double mops = ((double)nrop) * 1e3 / ((double)dt); + const bool done = damp_add_test(d, mops); + char buf[64]; + if (v[XSA]) { + sprintf(buf, " set %zu %zu", v[XSA], v[XSS]); + } else if (v[XDA]) { + sprintf(buf, " del %zu %zu", v[XDA], v[XDS]); + } else if (v[XGA]) { + sprintf(buf, " get %zu %zu", v[XGA], v[XGS]); + } else if (v[XPA]) { + sprintf(buf, " pro %zu %zu", v[XPA], v[XPS]); + } else if (v[XNA]) { + sprintf(buf, " seeknext %zu %zu", v[XNA], v[XNS]); + } else if (v[XKA]) { + sprintf(buf, " seekskip %zu %zu", v[XKA], v[XKS]); + } else { + buf[0] = '\0'; + } + sprintf(out, "%s mops %.4lf avg %.4lf ravg %.4lf\n", buf, mops, damp_avg(d), damp_ravg(d)); + return done; +} + + static void +kvmap_batch_nop(const struct forker_worker_info * const info, + const struct priv * const priv, const u64 nr) +{ + (void)info; + (void)priv; + for (u64 i = 0; i < nr; i++) + cpu_pause(); +} + +// (parallel) load + static void +kvmap_batch_put_par(const struct forker_worker_info * const info, + const struct priv * const priv, const u64 nr) +{ + if (info->end_type != FORKER_END_COUNT) + return; + + const struct kvmap_api * const api = info->passdata[0]; + void * const ref = priv->ref; + struct kv * const tmp = priv->tmp; + const u64 nr1 = nr / info->conc; + const u64 id0 = nr1 * info->worker_id; + const u64 end = (info->worker_id == (info->conc - 1)) ? nr : (id0 + nr1); + u64 ss = 0; + for (u64 i = id0; i < end; i++) { + kv_refill_hex64_klen(tmp, i, priv->klen, NULL, 0); + tmp->vlen = priv->vlen; + if (kvmap_kv_put(api, ref, tmp)) + ss++; + } + vctr_add(info->vctr, XSA, end - id0); + vctr_add(info->vctr, XSS, ss); +} + +// (parallel) probe + static void +kvmap_batch_probe_par(const struct forker_worker_info * const info, + const struct priv * const priv, const u64 nr) +{ + if (info->end_type != FORKER_END_COUNT) + return; + + const struct kvmap_api * const api = info->passdata[0]; + void * const ref = priv->ref; + struct kv * const tmp = priv->tmp; + const u64 nr1 = nr / info->conc; + const u64 id0 = nr1 * info->worker_id; + const u64 end = (info->worker_id == (info->conc - 1)) ? nr : (id0 + nr1); + u64 ss = 0; + for (u64 i = id0; i < end; i++) { + kv_refill_hex64_klen(tmp, i, priv->klen, NULL, 0); + if (kvmap_kv_probe(api, ref, tmp)) + ss++; + } + vctr_add(info->vctr, XPA, end - id0); + vctr_add(info->vctr, XPS, ss); +} + +// (parallel) get + static void +kvmap_batch_get_par(const struct forker_worker_info * const info, + const struct priv * const priv, const u64 nr) +{ + if (info->end_type != FORKER_END_COUNT) + return; + + const struct kvmap_api * const api = info->passdata[0]; + void * const ref = priv->ref; + struct kv * const tmp = priv->tmp; + const u64 nr1 = nr / info->conc; + const u64 id0 = nr1 * info->worker_id; + const u64 end = (info->worker_id == (info->conc - 1)) ? nr : (id0 + nr1); + u64 ss = 0; + for (u64 i = id0; i < end; i++) { + kv_refill_hex64_klen(tmp, i, priv->klen, NULL, 0); + if (kvmap_kv_get(api, ref, tmp, priv->out)) + ss++; + } + vctr_add(info->vctr, XGA, end - id0); + vctr_add(info->vctr, XGS, ss); +} + +// (parallel) del + static void +kvmap_batch_del_par(const struct forker_worker_info * const info, + const struct priv * const priv, const u64 nr) +{ + if (info->end_type != FORKER_END_COUNT) + return; + + const struct kvmap_api * const api = info->passdata[0]; + void * const ref = priv->ref; + struct kv * const tmp = priv->tmp; + const u64 nr1 = nr / info->conc; + const u64 id0 = nr1 * info->worker_id; + const u64 end = (info->worker_id == (info->conc - 1)) ? nr : (id0 + nr1); + u64 ss = 0; + for (u64 i = id0; i < end; i++) { + kv_refill_hex64_klen(tmp, i, priv->klen, NULL, 0); + if (kvmap_kv_del(api, ref, tmp)) + ss++; + } + vctr_add(info->vctr, XDA, end - id0); + vctr_add(info->vctr, XDS, ss); +} + + static void +kvmap_batch_put(const struct forker_worker_info * const info, + const struct priv * const priv, const u64 nr) +{ + const struct kvmap_api * const api = info->passdata[0]; + void * const ref = priv->ref; + struct rgen * const gen = info->gen; + rgen_next_func next = info->rgen_next_write; + struct kv * const tmp = priv->tmp; + u64 ss = 0lu; + + for (u64 i = 0; i < nr; i++) { + kv_refill_hex64_klen(tmp, next(gen), priv->klen, NULL, 0); + tmp->vlen = priv->vlen; + if (kvmap_kv_put(api, ref, tmp)) + ss++; + } + vctr_add(info->vctr, XSA, nr); + vctr_add(info->vctr, XSS, ss); +} + + static void +kvmap_batch_del(const struct forker_worker_info * const info, + const struct priv * const priv, const u64 nr) +{ + const struct kvmap_api * const api = info->passdata[0]; + void * const ref = priv->ref; + struct rgen * const gen = info->gen; + rgen_next_func next = info->rgen_next_write; + struct kv * const tmp = priv->tmp; + u64 ss = 0lu; + + for (u64 i = 0; i < nr; i++) { + kv_refill_hex64_klen(tmp, next(gen), priv->klen, NULL, 0); + if (kvmap_kv_del(api, ref, tmp)) + ss++; + } + vctr_add(info->vctr, XDA, nr); + vctr_add(info->vctr, XDS, ss); +} + + static void +kvmap_batch_get(const struct forker_worker_info * const info, + const struct priv * const priv, const u64 nr) +{ + const struct kvmap_api * const api = info->passdata[0]; + void * const ref = priv->ref; + struct rgen * const gen = info->gen; + rgen_next_func next = info->rgen_next; + struct kv * const tmp = priv->tmp; + u64 ss = 0lu; + + for (u64 i = 0; i < nr; i++) { + kv_refill_hex64_klen(tmp, next(gen), priv->klen, NULL, 0); + if (kvmap_kv_get(api, ref, tmp, priv->out)) + ss++; + } + vctr_add(info->vctr, XGA, nr); + vctr_add(info->vctr, XGS, ss); +} + + static void +kvmap_batch_probe(const struct forker_worker_info * const info, + const struct priv * const priv, const u64 nr) +{ + const struct kvmap_api * const api = info->passdata[0]; + void * const ref = priv->ref; + struct rgen * const gen = info->gen; + rgen_next_func next = info->rgen_next; + struct kv * const tmp = priv->tmp; + u64 ss = 0lu; + + for (u64 i = 0; i < nr; i++) { + kv_refill_hex64_klen(tmp, next(gen), priv->klen, NULL, 0); + if (kvmap_kv_probe(api, ref, tmp)) + ss++; + } + vctr_add(info->vctr, XPA, nr); + vctr_add(info->vctr, XPS, ss); +} + + static void +kvmap_batch_seek_next(const struct forker_worker_info * const info, + const struct priv * const priv, const u64 nr) +{ + const struct kvmap_api * const api = info->passdata[0]; + void * const ref = priv->ref; + void * const iter = api->iter_create(ref); + const u32 nscan = priv->nscan; + debug_assert(iter); + struct rgen * const gen = info->gen; + rgen_next_func next = info->rgen_next; + u64 ss = 0lu; + + for (u64 i = 0; i < nr; i++) { + kv_refill_hex64_klen(priv->tmp, next(gen), priv->klen, NULL, 0); + kvmap_kv_iter_seek(api, iter, priv->tmp); + for (u32 j = 0; j < nscan; j++) + api->iter_next(iter, priv->out); + if (api->iter_valid(iter)) + ss++; + } + vctr_add(info->vctr, XNA, nr); + vctr_add(info->vctr, XNS, ss); + api->iter_destroy(iter); +} + + static void +kvmap_batch_seek_skip(const struct forker_worker_info * const info, + const struct priv * const priv, const u64 nr) +{ + const struct kvmap_api * const api = info->passdata[0]; + void * const ref = priv->ref; + void * const iter = api->iter_create(ref); + const u32 nscan = priv->nscan; + debug_assert(iter); + struct rgen * const gen = info->gen; + rgen_next_func next = info->rgen_next; + u64 ss = 0lu; + + for (u64 i = 0; i < nr; i++) { + kv_refill_hex64_klen(priv->tmp, next(gen), priv->klen, NULL, 0); + kvmap_kv_iter_seek(api, iter, priv->tmp); + api->iter_skip(iter, nscan); + if (api->iter_peek(iter, priv->out)) + ss++; + } + vctr_add(info->vctr, XKA, nr); + vctr_add(info->vctr, XKS, ss); + api->iter_destroy(iter); +} + + static void * +kvmap_worker(void * const ptr) +{ + struct forker_worker_info * const info = (typeof(info))ptr; + srandom_u64(info->seed); + + const char op = info->argv[0][0]; + typeof(kvmap_batch_probe) * batch_func = NULL; + switch (op) { + case 's': batch_func = kvmap_batch_put; break; + case 'd': batch_func = kvmap_batch_del; break; + case 'p': batch_func = kvmap_batch_probe; break; + case 'g': batch_func = kvmap_batch_get; break; + case 'n': batch_func = kvmap_batch_seek_next; break; + case 'k': batch_func = kvmap_batch_seek_skip; break; + case 'S': batch_func = kvmap_batch_put_par; break; + case 'D': batch_func = kvmap_batch_del_par; break; + case 'P': batch_func = kvmap_batch_probe_par; break; + case 'G': batch_func = kvmap_batch_get_par; break; + default: batch_func = kvmap_batch_nop; break; + } + + struct priv p; + p.klen = a2u32(info->argv[1]); + p.vlen = a2u32(info->argv[2]); // vlen/nscan + const struct kvmap_api * const api = info->passdata[0]; + p.ref = kvmap_ref(api, info->passdata[1]); + const u64 outlen = sizeof(struct kv) + p.klen + p.vlen + 4096; + p.tmp = yalloc(outlen); + debug_assert(p.tmp); + memset(p.tmp, 0, outlen); + p.out = yalloc(outlen); + debug_assert(p.out); + if (info->end_type == FORKER_END_TIME) { + do { + batch_func(info, &p, 1lu << 14); // batch size + } while (time_nsec() < info->end_magic); + } else if (info->end_type == FORKER_END_COUNT) { + batch_func(info, &p, info->end_magic); + } + kvmap_unref(api, p.ref); + free(p.out); + free(p.tmp); + return NULL; +} + +#define NARGS ((3)) + static void +dbtest_help_message(void) +{ + fprintf(stderr, "%s Usage: {api ... {rgen ... {pass ...}}}\n", __func__); + kvmap_api_helper_message(); + forker_passes_message(); + fprintf(stderr, "%s dbtest wargs[%d]: \n", __func__, NARGS); + fprintf(stderr, "%s s:set d:del g:get p:probe n:seeknext k:seekskip\n", __func__); + fprintf(stderr, "%s S:set D:del G:get P:probe (auto-parallel: magic-type=1; magic=nr_kvs; rgen ignored)\n", __func__); +} + + static int +test_kvmap(const int argc, char ** const argv) +{ + const struct kvmap_api * api = NULL; + void * map = NULL; + const int n1 = kvmap_api_helper(argc, argv, NULL, &api, &map); + if (n1 < 0) + return n1; + + char *pref[64] = {}; + memcpy(pref, argv, sizeof(pref[0]) * (size_t)n1); + pref[n1] = NULL; + + struct pass_info pi = {}; + pi.passdata[0] = (void *)api; + pi.passdata[1] = map; + pi.vctr_size = VCTRSZ; + pi.wf = kvmap_worker; + pi.af = kvmap_analyze; + const int n2 = forker_passes(argc - n1, argv + n1, pref, &pi, NARGS); + + if (api->fprint) + api->fprint(map, stderr); + + api->destroy(map); + if (n2 < 0) { + return n2; + } else { + return n1 + n2; + } +} + + int +main(int argc, char ** argv) +{ + if (argc < 3) { + dbtest_help_message(); + exit(0); + } + + const bool r = forker_main(argc - 1, argv + 1, test_kvmap); + if (r == false) + dbtest_help_message(); + return 0; +} diff --git a/c/ycsbtest.c b/c/ycsbtest.c new file mode 100644 index 0000000..1c3889d --- /dev/null +++ b/c/ycsbtest.c @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2016--2021 Wu, Xingbo + * + * All rights reserved. No warranty, explicit or implicit, provided. + */ +#define _GNU_SOURCE + +#include "lib.h" +#include "kv.h" + +struct priv { + u32 klen; + u32 vlen; + u32 nscan; + u32 cget; + u32 cset; + u32 cupd; + u32 cscn; + void * ref; + void * iter; + struct kv * tmp; + struct kv * out; + struct kv ** kvs; +}; + +// load (use set) same to insert for latest dist. +#define XSA ((0)) +#define XSS ((1)) +// update (use merge), same to rmw (workload f, rarely mentioned) +#define XMA ((2)) +#define XMS ((3)) +// read (get) +#define XGA ((4)) +#define XGS ((5)) +// scan (seek-next) +#define XNA ((6)) +#define XNS ((7)) +// total number +#define XNR ((8)) + + static bool +kvmap_analyze(void * const passdata[2], const u64 dt, const struct vctr * const va, struct damp * const d, char * const out) +{ + (void)passdata; + size_t v[XNR]; + for (u64 i = 0; i < XNR; i++) + v[i] = vctr_get(va, i); + + const u64 nrop = v[XSA] + v[XMA] + v[XGA] + v[XNA]; + const double mops = ((double)nrop) * 1e3 / ((double)dt); + const bool done = damp_add_test(d, mops); + const char * const pat = " set %zu %zu upd %zu %zu get %zu %zu scan %zu %zu mops %.4lf avg %.4lf ravg %.4lf\n"; + sprintf(out, pat, v[XSA], v[XSS], v[XMA], v[XMS], v[XGA], v[XGS], v[XNA], v[XNS], mops, damp_avg(d), damp_ravg(d)); + return done; +} + + static struct kv * +kvmap_merge_dummy(struct kv * const key0, void * const priv) +{ + (void)key0; + return (struct kv *)priv; +} + + static void +kvmap_batch(const struct forker_worker_info * const info, + const struct priv * const priv, const u64 nr) +{ + const struct kvmap_api * const api = (typeof(api))info->passdata[0]; + void * const ref = priv->ref; + struct vctr * const v = info->vctr; + struct rgen * const gen = info->gen; + rgen_next_func next = info->rgen_next; + rgen_next_func next_write = info->rgen_next_write; + struct kv * const tmp = priv->tmp; + + for (u64 i = 0; i < nr; i++) { + const u32 p = random_u64() & 0xffffu; + if (p < priv->cget) { // GET + kv_refill_hex64_klen(tmp, next(gen), priv->klen, NULL, 0); + vctr_add1(v, XGA); + if (kvmap_kv_get(api, ref, tmp, priv->out)) + vctr_add1(v, XGS); + + } else if (p < priv->cscn) { // SCAN + kv_refill_hex64_klen(tmp, next(gen), priv->klen, NULL, 0); + vctr_add1(v, XNA); + + void * const iter = priv->iter; + debug_assert(iter); + + kvmap_kv_iter_seek(api, iter, tmp); + for (u32 k = 0; k < priv->nscan; k++) + api->iter_next(iter, priv->out); + if (api->iter_valid(iter)) + vctr_add1(v, XNS); + + // may need to park + if (api->iter_park) + api->iter_park(iter); + + } else if (p < priv->cset) { // SET + kv_refill_hex64_klen(tmp, next_write(gen), priv->klen, NULL, 0); + tmp->vlen = priv->vlen; + vctr_add1(v, XSA); + if (kvmap_kv_put(api, ref, tmp)) + vctr_add1(v, XSS); + + } else { // UPDATE (RMW) + kv_refill_hex64_klen(tmp, next_write(gen), priv->klen, NULL, 0); + tmp->vlen = priv->vlen; + vctr_add1(v, XMA); + if (api->merge) { // use merge() + if (kvmap_kv_merge(api, ref, tmp, kvmap_merge_dummy, tmp)) + vctr_add1(v, XMS); + } else { // GET & PUT + (void)kvmap_kv_get(api, ref, tmp, priv->out); // read + if (kvmap_kv_put(api, ref, tmp)) // write + vctr_add1(v, XMS); + } + } + } +} + + static void * +kvmap_worker(void * const ptr) +{ + struct forker_worker_info * const info = (typeof(info))ptr; + srandom_u64(info->seed); + const struct kvmap_api * const api = (typeof(api))info->passdata[0]; + + struct priv p = {}; + const u32 pset = a2u32(info->argv[0]); + const u32 pupd = a2u32(info->argv[1]); + const u32 pget = a2u32(info->argv[2]); + const u32 pscn = a2u32(info->argv[3]); + // scaled to 65536 + p.cget = pget * 65536 / 100; // 1st + p.cscn = (pget + pscn) * 65536 / 100; // 2nd + p.cset = (pget + pscn + pset) * 65536 / 100; // 3rd + p.cupd = 65536; // not used + (void)pupd; + + p.klen = a2u32(info->argv[4]); + p.vlen = a2u32(info->argv[5]); + p.nscan = a2u32(info->argv[6]); + p.ref = kvmap_ref(api, info->passdata[1]); + + if (pscn) { + p.iter = api->iter_create(p.ref); + if (api->iter_park) + api->iter_park(p.iter); + } + + const u64 outlen = sizeof(struct kv) + p.klen + p.vlen + 4096; + p.tmp = yalloc(outlen); + debug_assert(p.tmp); + memset(p.tmp, 0, outlen); + p.out = yalloc(outlen); + debug_assert(p.out); + + if (info->end_type == FORKER_END_TIME) { + do { + kvmap_batch(info, &p, 1lu << 14); + } while (time_nsec() < info->end_magic); + } else if (info->end_type == FORKER_END_COUNT) { + kvmap_batch(info, &p, info->end_magic); + } + + if (pscn) + api->iter_destroy(p.iter); + + kvmap_unref(api, p.ref); + free(p.tmp); + free(p.out); + return NULL; +} + +#define NARGS ((7)) + static void +maptest_help_message(void) +{ + fprintf(stderr, "%s Usage: {api ... {rgen ... {pass ...}}}\n", __func__); + kvmap_api_helper_message(); + forker_passes_message(); + fprintf(stderr, "%s wargs[%d]: \n", __func__, NARGS); + fprintf(stderr, "%s load kv samples at cpu: MAPTEST_KVLOAD_CPU=; default:1\n", __func__); +} + + static int +test_kvmap(const int argc, char ** const argv) +{ + const struct kvmap_api * api = NULL; + void * map = NULL; + const int n1 = kvmap_api_helper(argc, argv, NULL, &api, &map); + if (n1 < 0) + return n1; + + char *pref[64] = {}; + for (int i = 0; i < n1; i++) { + pref[i] = argv[i]; + } + pref[n1] = NULL; + + struct pass_info pi = {}; + pi.passdata[0] = (void *)api; + pi.passdata[1] = map; + pi.vctr_size = XNR; + pi.wf = kvmap_worker; + pi.af = kvmap_analyze; + const int n2 = forker_passes(argc - n1, argv + n1, pref, &pi, NARGS); + + if (api->fprint) + api->fprint(map, stderr); + + api->destroy(map); + if (n2 < 0) { + return n2; + } else { + return n1 + n2; + } +} + + int +main(int argc, char ** argv) +{ + if (argc < 3) { + maptest_help_message(); + exit(0); + } + + const bool r = forker_main(argc - 1, argv + 1, test_kvmap); + if (r == false) + maptest_help_message(); + return 0; +}