edits
authorweiyu <weiyuluo1232@gmail.com>
Wed, 9 Dec 2020 06:35:17 +0000 (22:35 -0800)
committerweiyu <weiyuluo1232@gmail.com>
Wed, 9 Dec 2020 06:35:17 +0000 (22:35 -0800)
firefox-related/icu.m4 [new file with mode: 0644]
firefox-related/jsshell-c11tester.sh [new file with mode: 0644]
gdax-orderbook-hpp/demo/demo.cpp
iris/src/base_logger.cpp
iris/tests/test_lfringbuffer.cpp
jsbench-2013.1/harness.py
mabain/examples/Makefile
mabain/examples/mb_multi_thread_insert_test.cpp
silo/Makefile
silo/compile.sh
silo/third-party/lz4/Makefile

diff --git a/firefox-related/icu.m4 b/firefox-related/icu.m4
new file mode 100644 (file)
index 0000000..0b30df9
--- /dev/null
@@ -0,0 +1,111 @@
+dnl This Source Code Form is subject to the terms of the Mozilla Public
+dnl License, v. 2.0. If a copy of the MPL was not distributed with this
+dnl file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+dnl Set the MOZ_ICU_VERSION variable to denote the current version of the
+dnl ICU library, as well as a few other things.
+
+AC_DEFUN([MOZ_CONFIG_ICU], [
+
+ICU_LIB_NAMES=
+MOZ_SYSTEM_ICU=
+MOZ_ARG_WITH_BOOL(system-icu,
+[  --with-system-icu
+                          Use system ICU (located with pkgconfig)],
+    MOZ_SYSTEM_ICU=1)
+
+if test -n "$MOZ_SYSTEM_ICU"; then
+    PKG_CHECK_MODULES(MOZ_ICU, icu-i18n >= 50.1)
+    CFLAGS="$CFLAGS $MOZ_ICU_CFLAGS"
+    CXXFLAGS="$CXXFLAGS $MOZ_ICU_CFLAGS"
+fi
+
+AC_SUBST(MOZ_SYSTEM_ICU)
+
+MOZ_ARG_WITH_STRING(intl-api,
+[  --with-intl-api, --with-intl-api=build, --without-intl-api
+    Determine the status of the ECMAScript Internationalization API.  The first
+    (or lack of any of these) builds and exposes the API.  The second builds it
+    but doesn't use ICU or expose the API to script.  The third doesn't build
+    ICU at all.],
+    _INTL_API=$withval)
+
+ENABLE_INTL_API=
+EXPOSE_INTL_API=
+case "$_INTL_API" in
+no)
+    ;;
+build)
+    ENABLE_INTL_API=1
+    ;;
+yes)
+    ENABLE_INTL_API=1
+    EXPOSE_INTL_API=1
+    ;;
+*)
+    AC_MSG_ERROR([Invalid value passed to --with-intl-api: $_INTL_API])
+    ;;
+esac
+
+if test -n "$ENABLE_INTL_API"; then
+    USE_ICU=1
+fi
+
+if test -n "$EXPOSE_INTL_API"; then
+    AC_DEFINE(EXPOSE_INTL_API)
+fi
+
+if test -n "$ENABLE_INTL_API"; then
+    AC_DEFINE(ENABLE_INTL_API)
+fi
+
+dnl Settings for the implementation of the ECMAScript Internationalization API
+if test -n "$USE_ICU"; then
+    icudir="$_topsrcdir/intl/icu/source"
+    if test ! -d "$icudir"; then
+        icudir="$_topsrcdir/../../intl/icu/source"
+        if test ! -d "$icudir"; then
+            AC_MSG_ERROR([Cannot find the ICU directory])
+        fi
+    fi
+
+    version=`sed -n 's/^[[[:space:]]]*#[[:space:]]*define[[:space:]][[:space:]]*U_ICU_VERSION_MAJOR_NUM[[:space:]][[:space:]]*\([0-9][0-9]*\)[[:space:]]*$/\1/p' "$icudir/common/unicode/uvernum.h"`
+    if test x"$version" = x; then
+       AC_MSG_ERROR([cannot determine icu version number from uvernum.h header file $lineno])
+    fi
+    MOZ_ICU_VERSION="$version"
+
+    # TODO: the l is actually endian-dependent
+    # We could make this set as 'l' or 'b' for little or big, respectively,
+    # but we'd need to check in a big-endian version of the file.
+    ICU_DATA_FILE="icudt${version}l.dat"
+
+    dnl We won't build ICU data as a separate file when building
+    dnl JS standalone so that embedders don't have to deal with it.
+    dnl We also don't do it on Windows because sometimes the file goes
+    dnl missing -- possibly due to overzealous antivirus software? --
+    dnl which prevents the browser from starting up :(
+    if test -z "$JS_STANDALONE" -a -z "$MOZ_SYSTEM_ICU" -a "$OS_TARGET" != WINNT -a "$MOZ_WIDGET_TOOLKIT" != "android"; then
+        MOZ_ICU_DATA_ARCHIVE=1
+    else
+        MOZ_ICU_DATA_ARCHIVE=
+    fi
+fi
+
+AC_SUBST(MOZ_ICU_VERSION)
+AC_SUBST(ENABLE_INTL_API)
+AC_SUBST(USE_ICU)
+AC_SUBST(ICU_DATA_FILE)
+AC_SUBST(MOZ_ICU_DATA_ARCHIVE)
+
+if test -n "$USE_ICU" -a -z "$MOZ_SYSTEM_ICU"; then
+    if test -z "$YASM" -a -z "$GNU_AS" -a "$COMPILE_ENVIRONMENT"; then
+      AC_MSG_ERROR([Building ICU requires either yasm or a GNU assembler. If you do not have either of those available for this platform you must use --without-intl-api])
+    fi
+    dnl We build ICU as a static library.
+    AC_DEFINE(U_STATIC_IMPLEMENTATION)
+    dnl Source files that use ICU should have control over which parts of the ICU
+    dnl namespace they want to use.
+    AC_DEFINE(U_USING_ICU_NAMESPACE,0)
+fi
+])
diff --git a/firefox-related/jsshell-c11tester.sh b/firefox-related/jsshell-c11tester.sh
new file mode 100644 (file)
index 0000000..79524ae
--- /dev/null
@@ -0,0 +1,19 @@
+#! /bin/sh
+
+if [ -z $1 ] ; then
+    echo "usage: $0 <dirname>"
+elif [ -d $1 ] ; then
+    echo "directory $1 already exists"
+else
+    autoconf2.13
+    mkdir $1
+    cd $1
+    LLVM_ROOT="/scratch/llvm"
+    CC="clang" \
+    CXX="clang++" \
+    CFLAGS="-Xclang -load -Xclang /scratch/llvm/build/lib/libCDSPass.so -L/scratch/fuzzer/random-fuzzer -lmodel" \
+    CXXFLAGS="-Xclang -load -Xclang /scratch/llvm/build/lib/libCDSPass.so -L/scratch/fuzzer/random-fuzzer -lmodel" \
+    LDFLAGS="-Xclang -load -Xclang /scratch/llvm/build/lib/libCDSPass.so -L/scratch/fuzzer/random-fuzzer -lmodel" \
+            ../configure --disable-debug --enable-optimize="-O2 -gline-tables-only" --enable-llvm-hacks --disable-jemalloc
+    make -j 8
+fi
index 4d3dc61..bcef9f2 100644 (file)
@@ -1,9 +1,16 @@
 #include <chrono>
 #include <iostream>
 #include <thread>
+#include <atomic>
+#include <cstdlib>
 
 #include "gdax-orderbook.hpp"
 
+#define FACTOR 125
+#define HIST_SIZE 60
+
+std::atomic<bool> stop;
+
 void printBestBidAndOffer(GDAXOrderBook & book)
 {
     std::cout << "current best bid: " << book.bids.begin()->second << " @ $"
@@ -16,24 +23,30 @@ void printBestBidAndOffer(GDAXOrderBook & book)
 int main(int argc, char* argv[]) {
     GDAXOrderBook book("ETH-USD");
 
-    printBestBidAndOffer(book);
+//    printBestBidAndOffer(book);
 
-    size_t secondsToSleep = 5;
+    size_t secondsToSleep = 30;
+    if (argc == 2) {
+        secondsToSleep = atoi(argv[1]);
+    }
+/*
     std::cout << "waiting " << secondsToSleep << " seconds for the market to "
         "shift" << std::endl;
     std::this_thread::sleep_for(std::chrono::seconds(secondsToSleep));
 
     printBestBidAndOffer(book);
+*/
 
-    size_t histogram[30];
-    for ( auto bucket : histogram ) bucket = 0;
+    size_t histogram[HIST_SIZE];
+    for ( size_t i = 0; i < HIST_SIZE; i++ )
+        histogram[i] = 0;
 
     size_t numThreads = 5;
-    secondsToSleep = 10;
     std::cout << "running for " << secondsToSleep << " seconds, with " <<
         numThreads << " threads constantly iterating over the whole order "
         "book." << std::endl;
     bool keepIterating = true;
+    stop.store(false);
     {
         std::vector<std::future<void>> futures;
         for (size_t i = 0 ; i < numThreads ; ++i)
@@ -60,12 +73,19 @@ int main(int argc, char* argv[]) {
 
                     finish = std::chrono::steady_clock::now();
 
-                   int index =                         static_cast<size_t>(
+                    int index =
+                        static_cast<size_t>(
                             std::chrono::duration<double, std::milli>(
                                 std::chrono::steady_clock::now() - start
-                                                                     ).count()/500);
-
-                           histogram[index]++;
+                            ).count()/FACTOR
+                        );
+
+                    if (!stop.load()) {
+                        if (index > HIST_SIZE)
+                            fprintf(stderr, "index overflow!\n");
+                        else
+                            histogram[index]++;
+                    }
                 }
             }));
         }
@@ -73,6 +93,7 @@ int main(int argc, char* argv[]) {
         std::this_thread::sleep_for(std::chrono::seconds(secondsToSleep));
 
         keepIterating = false;
+        stop.store(true);
     }
 
     // find the largest histogram bucket so we can determine the scale factor
@@ -82,22 +103,25 @@ int main(int argc, char* argv[]) {
             size_t countOfBiggestBucket = 0;
             for ( size_t & i : histogram )
             {
-             countOfBiggestBucket = std::max(i, countOfBiggestBucket);
+          countOfBiggestBucket = std::max(i, countOfBiggestBucket);
             }
             return (double)countOfBiggestBucket;
         }()/68.0; // 80 column display, minus chars used for row headers, =68
     std::cout << "histogram of times to iterate over the whole book:" << std::endl;
-    for ( int i=0 ; i < sizeof(histogram)/sizeof(histogram[0]) ; ++i )
+    for ( int i=0 ; i < HIST_SIZE ; ++i )
     {
         std::cout
-            << std::right << std::setw(3) << std::setfill(' ') << i*5
+            << std::right << std::setw(5) << std::setfill(' ') << i*FACTOR / 1000.0
             << "-"
-            << std::right << std::setw(3) << std::setfill(' ') << (i+1)*5-1
-            << " ms: ";
+            << std::right << std::setw(5) << std::setfill(' ') << (i+1)*FACTOR / 1000.0 - 0.001
+            << " s: ";
+        std::cout << histogram[i] << "\n";
+/*
         for ( int j=0 ; j<histogram[i]/scaleFactor ; ++j )
         {
             std::cout << "*";
         }
         std::cout << std::endl;
+*/
     }
 }
index a1a43ca..1357a9a 100644 (file)
@@ -30,7 +30,9 @@ thread_logqueue::thread_logqueue(thread_logqueue *head, size_t queue_size, size_
         p = head->next.load(std::memory_order_acquire);
         this->next.store(p, std::memory_order_release);
         // head might have been modified or deleted cas until this is inserted
-        if (head->next.compare_exchange_weak(p, this, std::memory_order_release)) {
+        // Use seq_cst ordering is a temporary fix.  C11tester did not implement pipe (used in notifier.cpp) so that
+        // writing to/reading from a pipe establish a happens-before relation
+        if (head->next.compare_exchange_weak(p, this, std::memory_order_seq_cst /*std::memory_order_release*/)) {
             return;
         }
     } while (true);
index 5f79354..e618be3 100644 (file)
@@ -16,7 +16,7 @@ iris::level_logger g_log(&writer, iris::TRACE);
 
 
 using namespace iris;
-#define ITERATIONS (int)1e7
+#define ITERATIONS (int) 1000000
 lfringbuffer rbuf(1024);
 struct buffer_t {
     char * b;
index fe39855..55a5638 100755 (executable)
@@ -29,6 +29,28 @@ import os
 import re
 import sys
 
+benchmarks = [
+    "amazon/chrome", "amazon/chrome-win", "amazon/firefox",
+    "amazon/firefox-win", "amazon/safari",
+    "facebook/chrome", "facebook/chrome-win", "facebook/firefox",
+    "facebook/firefox-win", "facebook/safari",
+    "google/chrome", "google/chrome-win", "google/firefox",
+    "google/firefox-win", "google/safari",
+    "twitter/chrome", "twitter/chrome-win", "twitter/firefox",
+    "twitter/firefox-win", "twitter/safari",
+    "yahoo/chrome", "yahoo/chrome-win", "yahoo/firefox",
+    "yahoo/firefox-win", "yahoo/safari",
+]
+
+modes = {
+    "*": ["urem"],
+    "amazon/firefox": ["urm"],
+    "amazon/firefox-win": ["urm"],
+    "google/firefox": ["uem"],
+    "twitter/chrome-win": ["rem"]
+};
+
+'''
 benchmarks = ["amazon/chrome", "amazon/firefox", "amazon/safari",
               "facebook/chrome", "facebook/firefox", "facebook/safari",
               "google/chrome", "google/firefox", "google/safari",
@@ -38,16 +60,18 @@ modes = {
     "*": ["urem"],
     "amazon/firefox": ["urm"],
     "google/firefox": ["uem"]
-}
-runcount = 25
-keepruns = 20
+}'''
 
-keepfrom = runcount - keepruns
+#runcount = 25
 
-if len(sys.argv) != 2:
-    print "Use: python harness.py <JS executable>"
+if len(sys.argv) != 3:
+    print "Use: python harness.py <JS executable> <number of runs>"
     exit(1)
 js = sys.argv[1]
+runcount = int(sys.argv[2])
+
+keepruns = 20
+keepfrom = runcount - keepruns
 
 # standard t-distribution for normally distributed samples
 tDistribution = [0, 0, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36, 2.31, 2.26,
@@ -109,7 +133,8 @@ for benchmark in benchmarks:
             # Now run it and get the results
             print(benchmark + " " + mode + " " + str(runno))
             res = os.popen(js + " " + benchmark + "/" + mode + ".js").read()
-            time = float(re.match("Time: ([0-9]*)ms", res).group(1))
+            time = float(re.findall("Time: ([0-9]*)ms", res)[0])
+#            time = float(re.match("Time: ([0-9]*)ms", res).group(1))
 
             if runno >= keepfrom:
                 results[benchmark][mode].append(time)
@@ -124,12 +149,11 @@ totals = {
     "runs": 0
 }
 
+power = 1.0 / len(benchmarks)
+
 for benchmark in benchmarks:
     sresults[benchmark] = {}
 
-
-    print("middle");
-
     bmodes = modes["*"]
     if benchmark in modes:
         bmodes = modes[benchmark]
@@ -141,39 +165,45 @@ for benchmark in benchmarks:
 
         sresult["mode"] = mode
 
-        mean = sresult["mean"] = sum(result) / len(result)
+        mean = sresult["mean"] = float(sum(result)) / len(result)
         stddev = sresult["stddev"] = math.sqrt(
             sum(
                 map(lambda e: math.pow(e - mean, 2), result)
-            ) / (len(result) - 1)
+            ) / float((len(result) - 1))
         )
 
-        sm = sresult["sm"] = stddev / mean
-        sem = sresult["sem"] = stddev / math.sqrt(len(result))
-        semm = sresult["semm"] = sem / mean
+        sm = sresult["sm"] = stddev / float(mean)
+        sem = sresult["sem"] = stddev / float(math.sqrt(len(result)))
+        semm = sresult["semm"] = sem / float(mean)
         ci = sresult["ci"] = tDist(len(result)) * sem
-        cim = sresult["cim"] = ci / mean
-
-        totals["mean"] *= mean
-        totals["stddev"] *= stddev
-        totals["sem"] *= sem
-        totals["ci"] *= ci
-
-power = 1 / totals["runs"]
-totals["mean"] = math.pow(totals["mean"], power)
-totals["stddev"] = math.pow(totals["stddev"], power)
-totals["sm"] = totals["stddev"] / totals["mean"]
-totals["sem"] = math.pow(totals["sem"], power)
-totals["semm"] = totals["sem"] / totals["mean"]
-totals["ci"] = math.pow(totals["ci"], power)
-totals["cim"] = totals["ci"] / totals["mean"]
+        cim = sresult["cim"] = ci / float(mean)
+
+        totals["mean"] *= math.pow(mean, power)
+
+        if (stddev == 0):
+            totals["stddev"] *= 1
+            print("stddev is 0")
+        else:
+            totals["stddev"] *= math.pow(stddev, power)
+        totals["sem"] *= math.pow(sem, power)
+        totals["ci"] *= math.pow(ci, power)
+
+#power = 1.0 / totals["runs"]
+#totals["mean"] = math.pow(totals["mean"], power)
+#totals["stddev"] = math.pow(totals["stddev"], power)
+#totals["sem"] = math.pow(totals["sem"], power)
+#totals["ci"] = math.pow(totals["ci"], power)
+
+totals["sm"] = totals["stddev"] / float(totals["mean"])
+totals["semm"] = totals["sem"] / float(totals["mean"])
+totals["cim"] = totals["ci"] / float(totals["mean"])
 
 totals["sm"] *= 100
 totals["semm"] *= 100
 totals["cim"] *= 100
 
 print "Final results:"
-print u"  %(mean)fms \u00b1 %(cim)f%% (lower is better)" % totals
+print "  %(mean)fms +- %(cim)f%% (lower is better)" % totals
 print "  Standard deviation = %(sm)f%% of mean" % totals
 print "  Standard error = %(semm)f%% of mean" % totals
 print "  %(runs)d runs" % {"runs": runcount}
@@ -188,7 +218,8 @@ for benchmark in benchmarks:
         bmodes = modes[benchmark]
 
     for mode in bmodes:
-        print u"  %(mode)s: %(mean)fms \u00b1 %(cim)f%% (stddev=%(sm)f%%, stderr=%(semm)f%%)" % sresults[benchmark][mode]
+        print "  %(mode)s: %(mean)fms +- %(cim)f%% (stddev=%(sm)f%%, stderr=%(semm)f%%)" % sresults[benchmark][mode]
+
 print ""
 
 print "Raw results:"
index 0dd2503..e279ae3 100644 (file)
@@ -1,5 +1,5 @@
 ifndef MABAIN_INSTALL_DIR
-MABAIN_INSTALL_DIR=/usr/local
+MABAIN_INSTALL_DIR=../src
 endif
 
 CPP=../../clang++
@@ -9,7 +9,7 @@ all: mb_insert_test mb_lookup_test mb_longest_prefix_test \
        mb_rc_test mb_multi_thread_insert_test mb_memory_only_test
 
 CFLAGS  = -I. -I$(MABAIN_INSTALL_DIR)/include -Wall -Werror -g -O0 -c -std=c++11
-LDFLAGS = -lpthread -lcrypto -L$(MABAIN_INSTALL_DIR)/lib -lmabain
+LDFLAGS = -lpthread -lcrypto -L$(MABAIN_INSTALL_DIR) -lmabain
 
 mb_insert_test: mb_insert_test.cpp
        $(CPP) $(CFLAGS) mb_insert_test.cpp
index 9a8b510..7e73ec0 100644 (file)
@@ -31,7 +31,7 @@ using namespace mabain;
 static int max_key = 1000000;
 static std::atomic<int> write_index;
 static bool stop_processing = false;
-static std::string mbdir = "/var/tmp/mabain_test/";
+static std::string mbdir = "./multi_test/";
 
 static void* insert_thread(void *arg)
 {
@@ -57,7 +57,7 @@ static void* insert_thread(void *arg)
     return NULL;
 }
 
-static void SetTestStatus(bool success)
+void SetTestStatus(bool success)
 {
     std::string cmd;
     if(success) {
@@ -104,7 +104,7 @@ int main(int argc, char *argv[])
         std::cout << "Setting number of keys to be " << max_key << "\n";
     }
 
-    SetTestStatus(false);
+//    SetTestStatus(false);
     mabain::DB::SetLogFile(mbdir + "/mabain.log");
 
     write_index.store(0, std::memory_order_release);
@@ -121,10 +121,6 @@ int main(int argc, char *argv[])
         }
     }
 
-    while(!stop_processing) {
-        usleep(5);
-    }
-
     for(int i = 0; i < nthread; i++) {
         pthread_join(pid[i], NULL);
     }
@@ -135,6 +131,6 @@ int main(int argc, char *argv[])
     Lookup();
 
     mabain::DB::CloseLogFile();
-    SetTestStatus(true);
+//    SetTestStatus(true);
     return 0;
 }
index 0c0f04f..1b00126 100644 (file)
@@ -2,6 +2,9 @@
 
 ### Options ###
 
+CC=../gcc
+CXX=../g++
+
 DEBUG ?= 0
 CHECK_INVARIANTS ?= 0
 
@@ -80,7 +83,7 @@ CXXFLAGS += -MD -Ithird-party/lz4 -DCONFIG_H=\"$(CONFIG_H)\"
 ifeq ($(DEBUG_S),1)
         CXXFLAGS += -fno-omit-frame-pointer -DDEBUG
 else
-        CXXFLAGS += -Werror -O2 -funroll-loops -fno-omit-frame-pointer
+        CXXFLAGS += -O2 -funroll-loops -fno-omit-frame-pointer
 endif
 ifeq ($(CHECK_INVARIANTS_S),1)
        CXXFLAGS += -DCHECK_INVARIANTS
index f046c00..ac0c2c9 100755 (executable)
@@ -1,6 +1,4 @@
 #!/bin/bash
-export PATH=`pwd`/..:$PATH
-source ../run
-MODE=perf CHECK_INVARIANTS=1 make -j
-MODE=perf CHECK_INVARIANTS=1 make -j dbtest
 
+#MODE=perf CHECK_INVARIANTS=0 make -j
+MODE=perf CHECK_INVARIANTS=0 USE_MALLOC_MODE=0 make -j dbtest 
index 9214589..c4e2284 100644 (file)
@@ -1,4 +1,4 @@
-CC=gcc
+CC=../../../gcc
 CFLAGS=-I. -std=c99 -Wall -W -Wundef -Wno-implicit-function-declaration
 
 OS := $(shell uname)