feat(bench press): add microIterations option

modules/benchpress/src/metric/perflog_metric.js

@@ -5,6 +5,7 @@ import { bind, OpaqueToken } from 'angular2/di';
 
 import { WebDriverExtension } from '../web_driver_extension';
 import { Metric } from '../metric';
+import { Options } from '../sample_options';
 
 /**
  * A metric that reads out the performance log
@@ -19,17 +20,25 @@ export class PerflogMetric extends Metric {
   _remainingEvents:List;
   _measureCount:int;
   _setTimeout:Function;
+  _microIterations:int;
 
-  constructor(driverExtension:WebDriverExtension, setTimeout:Function) {
+  /**
+   * @param driverExtension
+   * @param setTimeout
+   * @param microIterations Number of iterations that run inside the browser by user code.
+   *                        Used for micro benchmarks.
+   **/
+  constructor(driverExtension:WebDriverExtension, setTimeout:Function, microIterations:int) {
     super();
     this._driverExtension = driverExtension;
     this._remainingEvents = [];
     this._measureCount = 0;
     this._setTimeout = setTimeout;
+    this._microIterations = microIterations;
   }
 
   describe():StringMap {
-    return {
+    var res = {
       'script': 'script execution time in ms',
       'render': 'render time in ms',
       'gcTime': 'gc time in ms',
@@ -37,6 +46,10 @@ export class PerflogMetric extends Metric {
       'gcTimeInScript': 'gc time during script execution in ms',
       'gcAmountInScript': 'gc amount during script execution in kbytes'
     };
+    if (this._microIterations > 0) {
+      res['scriptMicroAvg'] = 'average script time for a micro iteration';
+    }
+    return res;
   }
 
   beginMeasure():Promise {
@@ -148,6 +161,9 @@ export class PerflogMetric extends Metric {
       }
     });
     result['script'] -= result['gcTimeInScript'];
+    if (this._microIterations > 0) {
+      result['scriptMicroAvg'] = result['script'] / this._microIterations;
+    }
     return isPresent(markStartEvent) && isPresent(markEndEvent) ? result : null;
   }
 
@@ -161,8 +177,9 @@ var _MARK_NAME_PREFIX = 'benchpress';
 var _SET_TIMEOUT = new OpaqueToken('PerflogMetric.setTimeout');
 var _BINDINGS = [
   bind(PerflogMetric).toFactory(
-    (driverExtension, setTimeout) => new PerflogMetric(driverExtension, setTimeout),
-    [WebDriverExtension, _SET_TIMEOUT]
+    (driverExtension, setTimeout, microIterations) => new PerflogMetric(driverExtension, setTimeout, microIterations),
+    [WebDriverExtension, _SET_TIMEOUT, Options.MICRO_ITERATIONS]
   ),
-  bind(_SET_TIMEOUT).toValue( (fn, millis) => PromiseWrapper.setTimeout(fn, millis) )
+  bind(_SET_TIMEOUT).toValue( (fn, millis) => PromiseWrapper.setTimeout(fn, millis) ),
+  bind(Options.MICRO_ITERATIONS).toValue(0)
 ];

modules/benchpress/src/runner.js

@@ -34,7 +34,7 @@ export class Runner {
     this._defaultBindings = defaultBindings;
   }
 
-  sample({id, execute, prepare, bindings}):Promise<SampleState> {
+  sample({id, execute, prepare, microIterations, bindings}):Promise<SampleState> {
     var sampleBindings = [
       _DEFAULT_BINDINGS,
       this._defaultBindings,
@@ -44,6 +44,9 @@ export class Runner {
     if (isPresent(prepare)) {
       ListWrapper.push(sampleBindings, bind(Options.PREPARE).toValue(prepare));
     }
+    if (isPresent(microIterations)) {
+      ListWrapper.push(sampleBindings, bind(Options.MICRO_ITERATIONS).toValue(microIterations));
+    }
     if (isPresent(bindings)) {
       ListWrapper.push(sampleBindings, bindings);
     }

modules/benchpress/src/sample_options.js

@@ -17,6 +17,12 @@ export class Options {
   static get CAPABILITIES() { return _CAPABILITIES; }
   // TODO(tbosch): use static initializer when our transpiler supports it
   static get USER_AGENT() { return _USER_AGENT; }
+  // TODO(tbosch): use static initializer when our transpiler supports it
+  /**
+   * Number of iterations that run inside the browser by user code.
+   * Used for micro benchmarks.
+   **/
+  static get MICRO_ITERATIONS() { return _MICRO_ITERATIONS; }
 }
 
 var _SAMPLE_ID = new OpaqueToken('Options.sampleId');
@@ -27,3 +33,4 @@ var _PREPARE = new OpaqueToken('Options.prepare');
 var _EXECUTE = new OpaqueToken('Options.execute');
 var _CAPABILITIES = new OpaqueToken('Options.capabilities');
 var _USER_AGENT = new OpaqueToken('Options.userAgent');
+var _MICRO_ITERATIONS = new OpaqueToken('Options.microIterations');

modules/benchpress/src/sampler.js

@@ -136,7 +136,10 @@ var _BINDINGS = [
       execute: execute,
       time: time
     }),
-    [WebDriverAdapter, WebDriverExtension, Metric, Reporter, Validator, Options.FORCE_GC, Options.PREPARE, Options.EXECUTE, _TIME]
+    [
+      WebDriverAdapter, WebDriverExtension, Metric, Reporter, Validator,
+      Options.FORCE_GC, Options.PREPARE, Options.EXECUTE, _TIME
+    ]
   ),
   bind(Options.FORCE_GC).toValue(false),
   bind(Options.PREPARE).toValue(false),

modules/benchpress/test/metric/perflog_metric_spec.js

@@ -2,8 +2,9 @@ import {ddescribe, describe, it, iit, xit, expect, beforeEach, afterEach} from '
 
 import { List, ListWrapper } from 'angular2/src/facade/collection';
 import { PromiseWrapper, Promise } from 'angular2/src/facade/async';
+import { isPresent } from 'angular2/src/facade/lang';
 
-import { Metric, PerflogMetric, WebDriverExtension, bind, Injector } from 'benchpress/common';
+import { Metric, PerflogMetric, WebDriverExtension, bind, Injector, Options } from 'benchpress/common';
 
 import { TraceEventFactory } from '../trace_event_factory';
 
@@ -11,16 +12,18 @@ export function main() {
   var commandLog;
   var eventFactory = new TraceEventFactory('timeline', 'pid0');
 
-  function createMetric(perfLogs) {
+  function createMetric(perfLogs, microIterations = 0) {
     commandLog = [];
-    return new Injector([
+    var bindings = [
       PerflogMetric.BINDINGS,
       bind(PerflogMetric.SET_TIMEOUT).toValue( (fn, millis) => {
         ListWrapper.push(commandLog, ['setTimeout', millis]);
         fn();
       }),
-      bind(WebDriverExtension).toValue(new MockDriverExtension(perfLogs, commandLog))
-    ]).get(PerflogMetric);
+      bind(WebDriverExtension).toValue(new MockDriverExtension(perfLogs, commandLog)),
+      bind(Options.MICRO_ITERATIONS).toValue(microIterations)
+    ];
+    return new Injector(bindings).get(PerflogMetric);
   }
 
   describe('perflog metric', () => {
@@ -151,10 +154,10 @@ export function main() {
 
     describe('aggregation', () => {
 
-      function aggregate(events) {
+      function aggregate(events, microIterations = 0) {
         ListWrapper.insert(events, 0, eventFactory.markStart('benchpress0', 0));
         ListWrapper.push(events, eventFactory.markEnd('benchpress0', 10));
-        var metric = createMetric([events]);
+        var metric = createMetric([events], microIterations);
         return metric
           .beginMeasure().then( (_) => metric.endMeasure(false) );
       }
@@ -252,6 +255,31 @@ export function main() {
         });
       });
 
+      describe('microIterations', () => {
+
+        it('should not report scriptMicroAvg if microIterations = 0', (done) => {
+          aggregate([
+            eventFactory.start('script', 0),
+            eventFactory.end('script', 5)
+          ], 0).then((data) => {
+            expect(isPresent(data['scriptMicroAvg'])).toBe(false);
+            done();
+          });
+        });
+
+        it('should report scriptMicroAvg', (done) => {
+          aggregate([
+            eventFactory.start('script', 0),
+            eventFactory.end('script', 5)
+          ], 4).then((data) => {
+            expect(data['script']).toBe(5);
+            expect(data['scriptMicroAvg']).toBe(5/4);
+            done();
+          });
+        });
+
+      });
+
       describe('gcTimeInScript / gcAmountInScript', () => {
 
         it('should detect gc during script execution with begin/end events', (done) => {

modules/benchpress/test/runner_spec.js

@@ -85,6 +85,13 @@ export function main() {
       });
     });
 
+    it('should bind Options.MICRO_ITERATIONS', (done) => {
+      createRunner().sample({id: 'someId', microIterations: 23}).then( (_) => {
+        expect(injector.get(Options.MICRO_ITERATIONS)).toEqual(23);
+        done();
+      });
+    });
+
     it('should overwrite bindings per sample call', (done) => {
       createRunner([
         bind(Options.DEFAULT_DESCRIPTION).toValue({'a': 1}),