X-Git-Url: http://plrg.eecs.uci.edu/git/?p=c11concurrency-benchmarks.git;a=blobdiff_plain;f=silo%2Fbenchmarks%2Fdbtest.cc;fp=silo%2Fbenchmarks%2Fdbtest.cc;h=0605654ad2f04715d6a6dc8b98fea3b7d88c4012;hp=0000000000000000000000000000000000000000;hb=19b84c667216ff74f1b747e18b5542444dc54716;hpb=72045f68e6d1ad46133bdc507b409b676ce64957 diff --git a/silo/benchmarks/dbtest.cc b/silo/benchmarks/dbtest.cc new file mode 100644 index 0000000..0605654 --- /dev/null +++ b/silo/benchmarks/dbtest.cc @@ -0,0 +1,407 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "../allocator.h" +#include "../stats_server.h" +#include "bench.h" +#include "bdb_wrapper.h" +#include "ndb_wrapper.h" +#include "ndb_wrapper_impl.h" +#include "kvdb_wrapper.h" +#include "kvdb_wrapper_impl.h" +#if !NO_MYSQL +#include "mysql_wrapper.h" +#endif + +using namespace std; +using namespace util; + +static vector +split_ws(const string &s) +{ + vector r; + istringstream iss(s); + copy(istream_iterator(iss), + istream_iterator(), + back_inserter>(r)); + return r; +} + +static size_t +parse_memory_spec(const string &s) +{ + string x(s); + size_t mult = 1; + if (x.back() == 'G') { + mult = static_cast(1) << 30; + x.pop_back(); + } else if (x.back() == 'M') { + mult = static_cast(1) << 20; + x.pop_back(); + } else if (x.back() == 'K') { + mult = static_cast(1) << 10; + x.pop_back(); + } + return strtoul(x.c_str(), nullptr, 10) * mult; +} + +int +main(int argc, char **argv) +{ + abstract_db *db = NULL; + void (*test_fn)(abstract_db *, int argc, char **argv) = NULL; + string bench_type = "ycsb"; + string db_type = "ndb-proto2"; + char *curdir = get_current_dir_name(); + string basedir = curdir; + string bench_opts; + size_t numa_memory = 0; + free(curdir); + int saw_run_spec = 0; + int nofsync = 0; + int do_compress = 0; + int fake_writes = 0; + int disable_gc = 0; + int disable_snapshots = 0; + vector logfiles; + vector> assignments; + string stats_server_sockfile; + while (1) { + static struct option long_options[] = + { + {"verbose" , no_argument , &verbose , 1} , + {"parallel-loading" , no_argument , &enable_parallel_loading , 1} , + {"pin-cpus" , no_argument , &pin_cpus , 1} , + {"slow-exit" , no_argument , &slow_exit , 1} , + {"retry-aborted-transactions" , no_argument , &retry_aborted_transaction , 1} , + {"backoff-aborted-transactions" , no_argument , &backoff_aborted_transaction , 1} , + {"bench" , required_argument , 0 , 'b'} , + {"scale-factor" , required_argument , 0 , 's'} , + {"num-threads" , required_argument , 0 , 't'} , + {"db-type" , required_argument , 0 , 'd'} , + {"basedir" , required_argument , 0 , 'B'} , + {"txn-flags" , required_argument , 0 , 'f'} , + {"runtime" , required_argument , 0 , 'r'} , + {"ops-per-worker" , required_argument , 0 , 'n'} , + {"bench-opts" , required_argument , 0 , 'o'} , + {"numa-memory" , required_argument , 0 , 'm'} , // implies --pin-cpus + {"logfile" , required_argument , 0 , 'l'} , + {"assignment" , required_argument , 0 , 'a'} , + {"log-nofsync" , no_argument , &nofsync , 1} , + {"log-compress" , no_argument , &do_compress , 1} , + {"log-fake-writes" , no_argument , &fake_writes , 1} , + {"disable-gc" , no_argument , &disable_gc , 1} , + {"disable-snapshots" , no_argument , &disable_snapshots , 1} , + {"stats-server-sockfile" , required_argument , 0 , 'x'} , + {"no-reset-counters" , no_argument , &no_reset_counters , 1} , + {0, 0, 0, 0} + }; + int option_index = 0; + int c = getopt_long(argc, argv, "b:s:t:d:B:f:r:n:o:m:l:a:x:", long_options, &option_index); + if (c == -1) + break; + + switch (c) { + case 0: + if (long_options[option_index].flag != 0) + break; + abort(); + break; + + case 'b': + bench_type = optarg; + break; + + case 's': + scale_factor = strtod(optarg, NULL); + ALWAYS_ASSERT(scale_factor > 0.0); + break; + + case 't': + nthreads = strtoul(optarg, NULL, 10); + ALWAYS_ASSERT(nthreads > 0); + break; + + case 'd': + db_type = optarg; + break; + + case 'B': + basedir = optarg; + break; + + case 'f': + txn_flags = strtoul(optarg, NULL, 10); + break; + + case 'r': + ALWAYS_ASSERT(!saw_run_spec); + saw_run_spec = 1; + runtime = strtoul(optarg, NULL, 10); + ALWAYS_ASSERT(runtime > 0); + run_mode = RUNMODE_TIME; + break; + + case 'n': + ALWAYS_ASSERT(!saw_run_spec); + saw_run_spec = 1; + ops_per_worker = strtoul(optarg, NULL, 10); + ALWAYS_ASSERT(ops_per_worker > 0); + run_mode = RUNMODE_OPS; + + case 'o': + bench_opts = optarg; + break; + + case 'm': + { + pin_cpus = 1; + const size_t m = parse_memory_spec(optarg); + ALWAYS_ASSERT(m > 0); + numa_memory = m; + } + break; + + case 'l': + logfiles.emplace_back(optarg); + break; + + case 'a': + assignments.emplace_back( + ParseCSVString>(optarg)); + break; + + case 'x': + stats_server_sockfile = optarg; + break; + + case '?': + /* getopt_long already printed an error message. */ + exit(1); + + default: + abort(); + } + } + + if (bench_type == "ycsb") + test_fn = ycsb_do_test; + else if (bench_type == "tpcc") + test_fn = tpcc_do_test; + else if (bench_type == "queue") + test_fn = queue_do_test; + else if (bench_type == "encstress") + test_fn = encstress_do_test; + else if (bench_type == "bid") + test_fn = bid_do_test; + else + ALWAYS_ASSERT(false); + + if (do_compress && logfiles.empty()) { + cerr << "[ERROR] --log-compress specified without logging enabled" << endl; + return 1; + } + + if (fake_writes && logfiles.empty()) { + cerr << "[ERROR] --log-fake-writes specified without logging enabled" << endl; + return 1; + } + + if (nofsync && logfiles.empty()) { + cerr << "[ERROR] --log-nofsync specified without logging enabled" << endl; + return 1; + } + + if (fake_writes && nofsync) { + cerr << "[WARNING] --log-nofsync has no effect with --log-fake-writes enabled" << endl; + } + +#ifndef ENABLE_EVENT_COUNTERS + if (!stats_server_sockfile.empty()) { + cerr << "[WARNING] --stats-server-sockfile with no event counters enabled is useless" << endl; + } +#endif + + // initialize the numa allocator + if (numa_memory > 0) { + const size_t maxpercpu = util::iceil( + numa_memory / nthreads, ::allocator::GetHugepageSize()); + numa_memory = maxpercpu * nthreads; + ::allocator::Initialize(nthreads, maxpercpu); + } + + const set can_persist({"ndb-proto2"}); + if (!logfiles.empty() && !can_persist.count(db_type)) { + cerr << "[ERROR] benchmark " << db_type + << " does not have persistence implemented" << endl; + return 1; + } + +#ifdef PROTO2_CAN_DISABLE_GC + const set has_gc({"ndb-proto1", "ndb-proto2"}); + if (disable_gc && !has_gc.count(db_type)) { + cerr << "[ERROR] benchmark " << db_type + << " does not have gc to disable" << endl; + return 1; + } +#else + if (disable_gc) { + cerr << "[ERROR] macro PROTO2_CAN_DISABLE_GC was not set, cannot disable gc" << endl; + return 1; + } +#endif + +#ifdef PROTO2_CAN_DISABLE_SNAPSHOTS + const set has_snapshots({"ndb-proto2"}); + if (disable_snapshots && !has_snapshots.count(db_type)) { + cerr << "[ERROR] benchmark " << db_type + << " does not have snapshots to disable" << endl; + return 1; + } +#else + if (disable_snapshots) { + cerr << "[ERROR] macro PROTO2_CAN_DISABLE_SNAPSHOTS was not set, cannot disable snapshots" << endl; + return 1; + } +#endif + + if (db_type == "bdb") { + const string cmd = "rm -rf " + basedir + "/db/*"; + // XXX(stephentu): laziness + int ret UNUSED = system(cmd.c_str()); + db = new bdb_wrapper("db", bench_type + ".db"); + } else if (db_type == "ndb-proto1") { + // XXX: hacky simulation of proto1 + db = new ndb_wrapper( + logfiles, assignments, !nofsync, do_compress, fake_writes); + transaction_proto2_static::set_hack_status(true); + ALWAYS_ASSERT(transaction_proto2_static::get_hack_status()); +#ifdef PROTO2_CAN_DISABLE_GC + if (!disable_gc) + transaction_proto2_static::InitGC(); +#endif + } else if (db_type == "ndb-proto2") { + db = new ndb_wrapper( + logfiles, assignments, !nofsync, do_compress, fake_writes); + ALWAYS_ASSERT(!transaction_proto2_static::get_hack_status()); +#ifdef PROTO2_CAN_DISABLE_GC + if (!disable_gc) + transaction_proto2_static::InitGC(); +#endif +#ifdef PROTO2_CAN_DISABLE_SNAPSHOTS + if (disable_snapshots) + transaction_proto2_static::DisableSnapshots(); +#endif + } else if (db_type == "kvdb") { + db = new kvdb_wrapper; + } else if (db_type == "kvdb-st") { + db = new kvdb_wrapper; +#if !NO_MYSQL + } else if (db_type == "mysql") { + string dbdir = basedir + "/mysql-db"; + db = new mysql_wrapper(dbdir, bench_type); +#endif + } else + ALWAYS_ASSERT(false); + +#ifdef DEBUG + cerr << "WARNING: benchmark built in DEBUG mode!!!" << endl; +#endif + +#ifdef CHECK_INVARIANTS + cerr << "WARNING: invariant checking is enabled - should disable for benchmark" << endl; +#ifdef PARANOID_CHECKING + cerr << " *** Paranoid checking is enabled ***" << endl; +#endif +#endif + + if (verbose) { + const unsigned long ncpus = coreid::num_cpus_online(); + cerr << "Database Benchmark:" << endl; + cerr << " pid: " << getpid() << endl; + cerr << "settings:" << endl; + cerr << " par-loading : " << enable_parallel_loading << endl; + cerr << " pin-cpus : " << pin_cpus << endl; + cerr << " slow-exit : " << slow_exit << endl; + cerr << " retry-txns : " << retry_aborted_transaction << endl; + cerr << " backoff-txns: " << backoff_aborted_transaction << endl; + cerr << " bench : " << bench_type << endl; + cerr << " scale : " << scale_factor << endl; + cerr << " num-cpus : " << ncpus << endl; + cerr << " num-threads : " << nthreads << endl; + cerr << " db-type : " << db_type << endl; + cerr << " basedir : " << basedir << endl; + cerr << " txn-flags : " << hexify(txn_flags) << endl; + if (run_mode == RUNMODE_TIME) + cerr << " runtime : " << runtime << endl; + else + cerr << " ops/worker : " << ops_per_worker << endl; +#ifdef USE_VARINT_ENCODING + cerr << " var-encode : yes" << endl; +#else + cerr << " var-encode : no" << endl; +#endif + +#ifdef USE_JEMALLOC + cerr << " allocator : jemalloc" << endl; +#elif defined USE_TCMALLOC + cerr << " allocator : tcmalloc" << endl; +#elif defined USE_FLOW + cerr << " allocator : flow" << endl; +#else + cerr << " allocator : libc" << endl; +#endif + if (numa_memory > 0) { + cerr << " numa-memory : " << numa_memory << endl; + } else { + cerr << " numa-memory : disabled" << endl; + } + cerr << " logfiles : " << logfiles << endl; + cerr << " assignments : " << assignments << endl; + cerr << " disable-gc : " << disable_gc << endl; + cerr << " disable-snapshots : " << disable_snapshots << endl; + cerr << " stats-server-sockfile: " << stats_server_sockfile << endl; + + cerr << "system properties:" << endl; + cerr << " btree_internal_node_size: " << concurrent_btree::InternalNodeSize() << endl; + cerr << " btree_leaf_node_size : " << concurrent_btree::LeafNodeSize() << endl; + +#ifdef TUPLE_PREFETCH + cerr << " tuple_prefetch : yes" << endl; +#else + cerr << " tuple_prefetch : no" << endl; +#endif + +#ifdef BTREE_NODE_PREFETCH + cerr << " btree_node_prefetch : yes" << endl; +#else + cerr << " btree_node_prefetch : no" << endl; +#endif + + } + + if (!stats_server_sockfile.empty()) { + stats_server *srvr = new stats_server(stats_server_sockfile); + thread(&stats_server::serve_forever, srvr).detach(); + } + + vector bench_toks = split_ws(bench_opts); + int argc = 1 + bench_toks.size(); + char *argv[argc]; + argv[0] = (char *) bench_type.c_str(); + for (size_t i = 1; i <= bench_toks.size(); i++) + argv[i] = (char *) bench_toks[i - 1].c_str(); + test_fn(db, argc, argv); + delete db; + return 0; +}