2 * Copyright (C) 2011, 2012 Purdue University
3 * Written by Gregor Richards
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
9 * 1. Redistributions of source code must retain the above copyright notice,
10 * this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25 * POSSIBILITY OF SUCH DAMAGE.
29 // put benchmarks here
30 var benchmarks = ["amazon/chrome", "amazon/chrome-win", "amazon/firefox",
31 "amazon/firefox-win", "amazon/safari", "facebook/chrome",
32 "facebook/chrome-win", "facebook/firefox",
33 "facebook/firefox-win", "facebook/safari",
34 "google/chrome", "google/chrome-win", "google/firefox",
35 "google/firefox-win", "google/safari", "twitter/chrome",
36 "twitter/chrome-win", "twitter/firefox",
37 "twitter/firefox-win", "twitter/safari", "yahoo/chrome",
38 "yahoo/chrome-win", "yahoo/firefox", "yahoo/firefox-win",
42 "amazon/firefox": ["urm"],
43 "amazon/firefox-win": ["urm"],
44 "google/firefox": ["uem"],
45 "twitter/chrome-win": ["rem"]
52 // fixups for old engines
53 if (!Array.prototype.reduce) {
54 Array.prototype.reduce = function(f, val) {
55 for (var i = 0; i < this.length; i++) {
56 val = f(val, this[i]);
69 function getModes(bm) {
70 if (bm in modes) return modes[bm];
74 // call this to either rerun or go to another page and come back
77 if (window.sessionStorage) {
78 // store this for the session and go 'round to another page to force uncaching on Chrome
79 sessionStorage.JSBNG_harnessState = JSON.stringify({
82 curBenchmark: curBenchmark,
85 window.location.href = window.location.href.replace(/\/[^\/]*$/, "/reload.html");
92 // load our current state and run
94 var gob = document.getElementById("go");
96 if (window.sessionStorage && "JSBNG_harnessState" in sessionStorage) {
97 var state = JSON.parse(sessionStorage.JSBNG_harnessState);
98 results = state.results;
99 curRun = state.curRun;
100 curBenchmark = state.curBenchmark;
101 curMode = state.curMode;
102 setTimeout(runBenchmark, 200);
103 gob.style.display = "none";
107 gob.onclick = function() {
108 gob.style.display = "none";
109 setTimeout(runBenchmark, 200);
111 if (/#run/.test(window.location.href)) {
112 gob.style.display = "none";
113 setTimeout(runBenchmark, 200);
117 function runBenchmark() {
118 var output = document.getElementById("output");
119 var bmframe = document.getElementById("bmframe");
122 if (curRun < minRuns) {
123 output.innerHTML = (curRun+1) + "/" + minRuns;
125 // check if our S/M is OK
126 var stats = handleResults(false);
127 output.innerHTML = (curRun+1) + " " + stats.cim + " (" + maxCIM + ")";
128 if (curRun >= maxRuns || stats.cim < maxCIM) {
129 bmframe.src = "about:blank";
130 bmframe.style.display = "none";
132 if (window.sessionStorage) delete sessionStorage.JSBNG_harnessState;
137 // get out our benchmark and mode
138 var benchmark = benchmarks[curBenchmark];
139 var modes = getModes(benchmark);
140 var mode = modes[curMode];
141 if (++curMode >= modes.length) {
145 if (curBenchmark >= benchmarks.length) {
150 // make sure we have the results space
151 if (!(benchmark in results)) results[benchmark] = {};
152 if (!(mode in results[benchmark])) results[benchmark][mode] = [];
154 // set up the receiver
155 if (/v/.test(mode)) {
156 // verification mode, we only care if there's an error
157 window.JSBNG_handleResult = function(res) {
159 if (!("errors" in results)) results.errors = [];
160 results.errors.push(benchmark + "." + mode + ": " + res.msg);
165 window.JSBNG_handleResult = function(res) {
167 results[benchmark][mode].push(res.time);
174 bmframe.src = benchmark + "/" + mode + ".html";
177 // handle all of our results
178 function handleResults(pr) {
179 var output = document.getElementById("output");
180 function print(str) {
181 output.appendChild(document.createTextNode(str));
182 output.appendChild(document.createElement("br"));
184 function printarr(arr) {
185 for (var i = 0; i < arr.length; i++) print(arr[i]);
187 function percent(num) {
188 return (num*100).toFixed(2) + "%";
191 // clear out the intermediate results
192 if (pr) output.innerHTML = "";
203 // stuff to print later
207 var spc = "\u00a0\u00a0";
208 var spc2 = spc + spc;
210 // calculate all the real results
211 for (var b = 0; b < benchmarks.length; b++) {
212 var benchmark = benchmarks[b];
213 var modes = getModes(benchmark);
215 presults.push(spc + benchmark + ":");
216 praw.push(spc + benchmark + ":");
219 for (var m = 0; m < modes.length; m++) {
221 var bmresults = results[benchmark][mode].slice(-keepRuns);
222 if (bmresults.length == 0) continue;
224 // get the raw results
225 var rr = spc2 + mode + ": [";
226 for (var i = 0; i < bmresults.length; i++) {
227 if (i != 0) rr += ", ";
231 if (pr) praw.push(rr);
233 // now get the stats for this run
234 var bmstats = stats(bmresults);
236 // mul it to the totals
237 totals.mean *= bmstats.mean;
238 totals.stddev *= bmstats.stddev;
239 totals.sem *= bmstats.sem;
240 totals.ci *= bmstats.ci;
244 if (pr) presults.push(spc2 + mode + ": " +
245 bmstats.mean.toFixed(2) + "ms ± " + percent(bmstats.cim) +
246 " (stddev=" + percent(bmstats.sm) + ", stderr=" +
247 percent(bmstats.semm) + ")");
256 // now calculate the totals
257 var power = 1 / totals.runs;
258 totals.mean = Math.pow(totals.mean, power);
259 totals.stddev = Math.pow(totals.stddev, power);
260 totals.sm = totals.stddev / totals.mean;
261 totals.sem = Math.pow(totals.sem, power);
262 totals.semm = totals.sem / totals.mean;
263 totals.ci = Math.pow(totals.ci, power);
264 totals.cim = totals.ci / totals.mean;
265 ptotals.push("Final results:");
266 ptotals.push(spc + totals.mean.toFixed(2) + "ms ± " + percent(totals.cim) + " (lower is better)");
267 ptotals.push(spc + "Standard deviation = " + percent(totals.sm) + " of mean");
268 ptotals.push(spc + "Standard error = " + percent(totals.semm) + " of mean");
269 if (totals.cim >= maxCIM)
270 ptotals.push(spc + "WARNING: These results are not trustworthy! After " + maxRuns + " runs, 95% confidence interval is still greater than " + percent(maxCIM) + " of the mean!");
272 ptotals.push(spc + curRun + " runs");
275 // if there are errors, mark those too
276 if ("errors" in results) {
277 ptotals.push("ERRORS:");
278 for (var i = 0; i < results.errors.length; i++) ptotals.push(spc + results.errors[i]);
283 // and print it all out
285 print("Result breakdown:");
287 print("Raw results:");
294 // standard t-distribution for normally distributed samples
295 var tDistribution = [NaN, NaN, 12.71, 4.30, 3.18, 2.78, 2.57, 2.45, 2.36,
296 2.31, 2.26, 2.23, 2.20, 2.18, 2.16, 2.14, 2.13, 2.12, 2.11, 2.10, 2.09,
297 2.09, 2.08, 2.07, 2.07, 2.06, 2.06, 2.06, 2.05, 2.05, 2.05, 2.04, 2.04,
298 2.04, 2.03, 2.03, 2.03, 2.03, 2.03, 2.02, 2.02, 2.02, 2.02, 2.02, 2.02,
299 2.02, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.01, 2.00, 2.00,
300 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00, 2.00,
301 2.00, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99,
302 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99, 1.99,
303 1.99, 1.99, 1.99, 1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
304 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
305 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
306 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
307 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98,
308 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.97, 1.97, 1.97, 1.97,
309 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
310 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
311 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
312 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
313 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
314 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
315 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
316 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
317 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
318 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
319 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
320 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
321 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
322 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
323 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
324 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
325 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
326 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
327 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
328 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
329 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
330 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
331 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
332 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
333 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97,
334 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.97, 1.96];
338 if (n >= tDistribution.length)
339 return tDistribution[tDistribution.length-1];
340 return tDistribution[n];
344 function stats(results) {
348 return arr.reduce(function(p, c) { return p + c; }, 0);
352 ret.mean = sum(results) / results.length;
355 ret.stddev = Math.sqrt(
357 results.map(function(e) { return Math.pow(e - ret.mean, 2); })
358 ) / (results.length - 1)
362 ret.sm = ret.stddev / ret.mean;
364 // sample SEM (stderr)
365 ret.sem = ret.stddev / Math.sqrt(results.length);
368 ret.semm = ret.sem / ret.mean;
370 // sample 95% confidence interval range
371 ret.ci = tDist(results.length) * ret.sem;
373 // sample 95% CI / mean
374 ret.cim = ret.ci / ret.mean;
379 window.onload = onload;