edits

[c11concurrency-benchmarks.git] / jsbench-2013.1 / harness.py

diff --git a/jsbench-2013.1/harness.py b/jsbench-2013.1/harness.py
index fe39855b5390b44dbedb5fdbe08b28b09d83444f..55a56380f5c3f740113f22d48b1353fadc57812f 100755 (executable)
--- a/jsbench-2013.1/harness.py
+++ b/jsbench-2013.1/harness.py
@@ -29,6 +29,28 @@ import os
 import re
 import sys
 
+benchmarks = [
+    "amazon/chrome", "amazon/chrome-win", "amazon/firefox",
+    "amazon/firefox-win", "amazon/safari",
+    "facebook/chrome", "facebook/chrome-win", "facebook/firefox",
+    "facebook/firefox-win", "facebook/safari",
+    "google/chrome", "google/chrome-win", "google/firefox",
+    "google/firefox-win", "google/safari",
+    "twitter/chrome", "twitter/chrome-win", "twitter/firefox",
+    "twitter/firefox-win", "twitter/safari",
+    "yahoo/chrome", "yahoo/chrome-win", "yahoo/firefox",
+    "yahoo/firefox-win", "yahoo/safari",
+]
+
+modes = {
+    "*": ["urem"],
+    "amazon/firefox": ["urm"],
+    "amazon/firefox-win": ["urm"],
+    "google/firefox": ["uem"],
+    "twitter/chrome-win": ["rem"]
+};
+
+'''
 benchmarks = ["amazon/chrome", "amazon/firefox", "amazon/safari",
               "facebook/chrome", "facebook/firefox", "facebook/safari",
               "google/chrome", "google/firefox", "google/safari",
@@ -38,16 +60,18 @@ modes = {
     "*": ["urem"],
     "amazon/firefox": ["urm"],
     "google/firefox": ["uem"]
-}
-runcount = 25
-keepruns = 20
+}'''
 
-keepfrom = runcount - keepruns
+#runcount = 25
 
-if len(sys.argv) != 2:
-    print "Use: python harness.py <JS executable>"
+if len(sys.argv) != 3:
+    print "Use: python harness.py <JS executable> <number of runs>"
     exit(1)
 js = sys.argv[1]
+runcount = int(sys.argv[2])
+
+keepruns = 20
+keepfrom = runcount - keepruns
 
 # standard t-distribution for normally distributed samples
 tDistribution = [0, 0, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36, 2.31, 2.26,
@@ -109,7 +133,8 @@ for benchmark in benchmarks:
             # Now run it and get the results
             print(benchmark + " " + mode + " " + str(runno))
             res = os.popen(js + " " + benchmark + "/" + mode + ".js").read()
-            time = float(re.match("Time: ([0-9]*)ms", res).group(1))
+            time = float(re.findall("Time: ([0-9]*)ms", res)[0])
+#            time = float(re.match("Time: ([0-9]*)ms", res).group(1))
 
             if runno >= keepfrom:
                 results[benchmark][mode].append(time)
@@ -124,12 +149,11 @@ totals = {
     "runs": 0
 }
 
+power = 1.0 / len(benchmarks)
+
 for benchmark in benchmarks:
     sresults[benchmark] = {}
 
-
-    print("middle");
-
     bmodes = modes["*"]
     if benchmark in modes:
         bmodes = modes[benchmark]
@@ -141,39 +165,45 @@ for benchmark in benchmarks:
 
         sresult["mode"] = mode
 
-        mean = sresult["mean"] = sum(result) / len(result)
+        mean = sresult["mean"] = float(sum(result)) / len(result)
         stddev = sresult["stddev"] = math.sqrt(
             sum(
                 map(lambda e: math.pow(e - mean, 2), result)
-            ) / (len(result) - 1)
+            ) / float((len(result) - 1))
         )
 
-        sm = sresult["sm"] = stddev / mean
-        sem = sresult["sem"] = stddev / math.sqrt(len(result))
-        semm = sresult["semm"] = sem / mean
+        sm = sresult["sm"] = stddev / float(mean)
+        sem = sresult["sem"] = stddev / float(math.sqrt(len(result)))
+        semm = sresult["semm"] = sem / float(mean)
         ci = sresult["ci"] = tDist(len(result)) * sem
-        cim = sresult["cim"] = ci / mean
-
-        totals["mean"] *= mean
-        totals["stddev"] *= stddev
-        totals["sem"] *= sem
-        totals["ci"] *= ci
-
-power = 1 / totals["runs"]
-totals["mean"] = math.pow(totals["mean"], power)
-totals["stddev"] = math.pow(totals["stddev"], power)
-totals["sm"] = totals["stddev"] / totals["mean"]
-totals["sem"] = math.pow(totals["sem"], power)
-totals["semm"] = totals["sem"] / totals["mean"]
-totals["ci"] = math.pow(totals["ci"], power)
-totals["cim"] = totals["ci"] / totals["mean"]
+        cim = sresult["cim"] = ci / float(mean)
+
+        totals["mean"] *= math.pow(mean, power)
+
+        if (stddev == 0):
+            totals["stddev"] *= 1
+            print("stddev is 0")
+        else:
+            totals["stddev"] *= math.pow(stddev, power)
+        totals["sem"] *= math.pow(sem, power)
+        totals["ci"] *= math.pow(ci, power)
+
+#power = 1.0 / totals["runs"]
+#totals["mean"] = math.pow(totals["mean"], power)
+#totals["stddev"] = math.pow(totals["stddev"], power)
+#totals["sem"] = math.pow(totals["sem"], power)
+#totals["ci"] = math.pow(totals["ci"], power)
+
+totals["sm"] = totals["stddev"] / float(totals["mean"])
+totals["semm"] = totals["sem"] / float(totals["mean"])
+totals["cim"] = totals["ci"] / float(totals["mean"])
 
 totals["sm"] *= 100
 totals["semm"] *= 100
 totals["cim"] *= 100
 
 print "Final results:"
-print u"  %(mean)fms \u00b1 %(cim)f%% (lower is better)" % totals
+print "  %(mean)fms +- %(cim)f%% (lower is better)" % totals
 print "  Standard deviation = %(sm)f%% of mean" % totals
 print "  Standard error = %(semm)f%% of mean" % totals
 print "  %(runs)d runs" % {"runs": runcount}
@@ -188,7 +218,8 @@ for benchmark in benchmarks:
         bmodes = modes[benchmark]
 
     for mode in bmodes:
-        print u"  %(mode)s: %(mean)fms \u00b1 %(cim)f%% (stddev=%(sm)f%%, stderr=%(semm)f%%)" % sresults[benchmark][mode]
+        print "  %(mode)s: %(mean)fms +- %(cim)f%% (stddev=%(sm)f%%, stderr=%(semm)f%%)" % sresults[benchmark][mode]
+
 print ""
 
 print "Raw results:"