feat(benchpress): initial support for firefox

Closes #2419

modules/benchpress/common.ts

@@ -11,6 +11,7 @@ export {JsonFileReporter} from './src/reporter/json_file_reporter';
 export {SampleDescription} from './src/sample_description';
 export {PerflogMetric} from './src/metric/perflog_metric';
 export {ChromeDriverExtension} from './src/webdriver/chrome_driver_extension';
+export {FirefoxDriverExtension} from './src/webdriver/firefox_driver_extension';
 export {IOsDriverExtension} from './src/webdriver/ios_driver_extension';
 export {Runner} from './src/runner';
 export {Options} from './src/common_options';

modules/benchpress/src/firefox_extension/data/installed_script.ts

@@ -1,11 +1,28 @@
 declare var exportFunction;
 declare var unsafeWindow;
 
-exportFunction(function() { (<any>self).port.emit('startProfiler'); }, unsafeWindow,
+exportFunction(function() {
-               {defineAs: "startProfiler"});
+  var curTime = unsafeWindow.performance.now();
+  (<any>self).port.emit('startProfiler', curTime);
+}, unsafeWindow, {defineAs: "startProfiler"});
 
-exportFunction(function(filePath) { (<any>self).port.emit('stopAndRecord', filePath); },
+exportFunction(function() { (<any>self).port.emit('stopProfiler'); }, unsafeWindow,
-               unsafeWindow, {defineAs: "stopAndRecord"});
+               {defineAs: "stopProfiler"});
+
+exportFunction(function(cb) {
+  (<any>self).port.once('perfProfile', cb);
+  (<any>self).port.emit('getProfile');
+}, unsafeWindow, {defineAs: "getProfile"});
 
 exportFunction(function() { (<any>self).port.emit('forceGC'); }, unsafeWindow,
                {defineAs: "forceGC"});
+
+exportFunction(function(name) {
+  var curTime = unsafeWindow.performance.now();
+  (<any>self).port.emit('markStart', name, curTime);
+}, unsafeWindow, {defineAs: "markStart"});
+
+exportFunction(function(name) {
+  var curTime = unsafeWindow.performance.now();
+  (<any>self).port.emit('markEnd', name, curTime);
+}, unsafeWindow, {defineAs: "markEnd"});

modules/benchpress/src/firefox_extension/lib/main.ts

@@ -1,52 +1,66 @@
 /// <reference path="../../../../angular2/typings/node/node.d.ts" />
 
-var file = require('sdk/io/file');
+var {Cc, Ci, Cu} = require('chrome');
-var {Cc, Ci, Cu} = require("chrome");
+var os = Cc['@mozilla.org/observer-service;1'].getService(Ci.nsIObserverService);
+var ParserUtil = require('./parser_util');
 
 class Profiler {
   private _profiler;
-  constructor() { this._profiler = Cc["@mozilla.org/tools/profiler;1"].getService(Ci.nsIProfiler); }
+  private _markerEvents: List<any>;
+  private _profilerStartTime: number;
 
-  start(entries, interval, features) {
+  constructor() { this._profiler = Cc['@mozilla.org/tools/profiler;1'].getService(Ci.nsIProfiler); }
+
+  start(entries, interval, features, timeStarted) {
     this._profiler.StartProfiler(entries, interval, features, features.length);
+    this._profilerStartTime = timeStarted;
+    this._markerEvents = [];
   }
 
   stop() { this._profiler.StopProfiler(); }
-  getProfileData() { return this._profiler.getProfileData(); }
-}
 
+  getProfilePerfEvents() {
+    var profileData = this._profiler.getProfileData();
+    var perfEvents = ParserUtil.convertPerfProfileToEvents(profileData);
+    perfEvents = this._mergeMarkerEvents(perfEvents);
+    perfEvents.sort(function(event1, event2) { return event1.ts - event2.ts; });  // Sort by ts
+    return perfEvents;
+  }
 
-function saveToFile(savePath: string, body: string) {
+  _mergeMarkerEvents(perfEvents: List<any>): List<any> {
-  var textWriter = file.open(savePath, 'w');
+    this._markerEvents.forEach(function(markerEvent) { perfEvents.push(markerEvent); });
-  textWriter.write(body);
+    return perfEvents;
-  textWriter.close();
+  }
+
+  addStartEvent(name: string, timeStarted: number) {
+    this._markerEvents.push({ph: 'b', ts: timeStarted - this._profilerStartTime, name: name});
+  }
+
+  addEndEvent(name: string, timeEnded: number) {
+    this._markerEvents.push({ph: 'e', ts: timeEnded - this._profilerStartTime, name: name});
+  }
 }
 
 function forceGC() {
   Cu.forceGC();
-  var os = Cc["@mozilla.org/observer-service;1"].getService(Ci.nsIObserverService);
+  os.notifyObservers(null, 'child-gc-request', null);
-  os.notifyObservers(null, "child-gc-request", null);
+};
-}
 
+var mod = require('sdk/page-mod');
+var data = require('sdk/self').data;
 var profiler = new Profiler();
-function startProfiler() {
-  profiler.start(/* = profiler memory */ 10000000, 1, ['leaf', 'js', "stackwalk", 'gc']);
-};
-function stopAndRecord(filePath) {
-  var profileData = profiler.getProfileData();
-  profiler.stop();
-  saveToFile(filePath, JSON.stringify(profileData, null, 2));
-};
-
-
-var mod = require("sdk/page-mod");
-var data = require("sdk/self").data;
 mod.PageMod({
   include: ['*'],
-  contentScriptFile: data.url("installed_script.js"),
+  contentScriptFile: data.url('installed_script.js'),
   onAttach: worker => {
-    worker.port.on('startProfiler', () => startProfiler());
+    worker.port.on('startProfiler',
-    worker.port.on('stopAndRecord', filePath => stopAndRecord(filePath));
+                   (timeStarted) => profiler.start(/* = profiler memory */ 1000000, 1,
-    worker.port.on('forceGC', () => forceGC());
+                                                   ['leaf', 'js', 'stackwalk', 'gc'], timeStarted));
+    worker.port.on('stopProfiler', () => profiler.stop());
+    worker.port.on('getProfile',
+                   () => worker.port.emit('perfProfile', profiler.getProfilePerfEvents()));
+    worker.port.on('forceGC', forceGC);
+    worker.port.on('markStart', (name, timeStarted) => profiler.addStartEvent(name, timeStarted));
+    worker.port.on('markEnd', (name, timeEnded) => profiler.addEndEvent(name, timeEnded));
   }
 });

modules/benchpress/src/firefox_extension/lib/parser_util.dart

@@ -0,0 +1,2 @@
+library benchpress.src.firefox_extension.lib.parser_util;
+//no dart implementation

modules/benchpress/src/firefox_extension/lib/parser_util.ts

@@ -0,0 +1,84 @@
+/// <reference path="../../../../angular2/typings/node/node.d.ts" />
+
+/**
+ * @param {Object} perfProfile The perf profile JSON object.
+ * @return {Array<Object>} An array of recognized events that are captured
+ *     within the perf profile.
+ */
+export function convertPerfProfileToEvents(perfProfile: any): List<any> {
+  var inProgressEvents = new Map();  // map from event name to start time
+  var finishedEvents = [];           // Array<Event> finished events
+  var addFinishedEvent = function(eventName, startTime, endTime) {
+    var categorizedEventName = categorizeEvent(eventName);
+    var args = undefined;
+    if (categorizedEventName == 'gc') {
+      // TODO: We cannot measure heap size at the moment
+      args = {usedHeapSize: 0};
+    }
+    if (startTime == endTime) {
+      // Finished instantly
+      finishedEvents.push({ph: 'X', ts: startTime, name: categorizedEventName, args: args});
+    } else {
+      // Has duration
+      finishedEvents.push({ph: 'B', ts: startTime, name: categorizedEventName, args: args});
+      finishedEvents.push({ph: 'E', ts: endTime, name: categorizedEventName, args: args});
+    }
+  };
+
+  var samples = perfProfile.threads[0].samples;
+  // In perf profile, firefox samples all the frames in set time intervals. Here
+  // we go through all the samples and construct the start and end time for each
+  // event.
+  for (var i = 0; i < samples.length; ++i) {
+    var sample = samples[i];
+    var sampleTime = sample.time;
+
+    // Add all the frames into a set so it's easier/faster to find the set
+    // differences
+    var sampleFrames = new Set();
+    sample.frames.forEach(function(frame) { sampleFrames.add(frame.location); });
+
+    // If an event is in the inProgressEvents map, but not in the current sample,
+    // then it must have just finished. We add this event to the finishedEvents
+    // array and remove it from the inProgressEvents map.
+    var previousSampleTime = (i == 0 ? /* not used */ -1 : samples[i - 1].time);
+    inProgressEvents.forEach(function(startTime, eventName) {
+      if (!(sampleFrames.has(eventName))) {
+        addFinishedEvent(eventName, startTime, previousSampleTime);
+        inProgressEvents.delete(eventName);
+      }
+    });
+
+    // If an event is in the current sample, but not in the inProgressEvents map,
+    // then it must have just started. We add this event to the inProgressEvents
+    // map.
+    sampleFrames.forEach(function(eventName) {
+      if (!(inProgressEvents.has(eventName))) {
+        inProgressEvents.set(eventName, sampleTime);
+      }
+    });
+  }
+
+  // If anything is still in progress, we need to included it as a finished event
+  // since recording ended.
+  var lastSampleTime = samples[samples.length - 1].time;
+  inProgressEvents.forEach(function(startTime, eventName) {
+    addFinishedEvent(eventName, startTime, lastSampleTime);
+  });
+
+  // Remove all the unknown categories.
+  return finishedEvents.filter(function(event) { return event.name != 'unknown'; });
+}
+
+// TODO: this is most likely not exhaustive.
+export function categorizeEvent(eventName: string): string {
+  if (eventName.indexOf('PresShell::Paint') > -1) {
+    return 'render';
+  } else if (eventName.indexOf('FirefoxDriver.prototype.executeScript') > -1) {
+    return 'script';
+  } else if (eventName.indexOf('forceGC') > -1) {
+    return 'gc';
+  } else {
+    return 'unknown';
+  }
+}

modules/benchpress/src/firefox_extension/package.json

@@ -1,5 +1 @@
-{
+{ "version" : "0.0.1", "main" : "lib/main.js", "name" : "ffperf-addon" }
-    "version": "0.0.1", 
-    "main": "lib/main.js",
-    "name": "ffperf-addon"
-}

modules/benchpress/src/runner.ts

@@ -12,6 +12,7 @@ import {Validator} from './validator';
 import {PerflogMetric} from './metric/perflog_metric';
 import {MultiMetric} from './metric/multi_metric';
 import {ChromeDriverExtension} from './webdriver/chrome_driver_extension';
+import {FirefoxDriverExtension} from './webdriver/firefox_driver_extension';
 import {IOsDriverExtension} from './webdriver/ios_driver_extension';
 import {WebDriverExtension} from './web_driver_extension';
 import {SampleDescription} from './sample_description';
@@ -62,6 +63,7 @@ var _DEFAULT_BINDINGS = [
   RegressionSlopeValidator.BINDINGS,
   SizeValidator.BINDINGS,
   ChromeDriverExtension.BINDINGS,
+  FirefoxDriverExtension.BINDINGS,
   IOsDriverExtension.BINDINGS,
   PerflogMetric.BINDINGS,
   SampleDescription.BINDINGS,
@@ -70,7 +72,7 @@ var _DEFAULT_BINDINGS = [
 
   Reporter.bindTo(MultiReporter),
   Validator.bindTo(RegressionSlopeValidator),
-  WebDriverExtension.bindTo([ChromeDriverExtension, IOsDriverExtension]),
+  WebDriverExtension.bindTo([ChromeDriverExtension, FirefoxDriverExtension, IOsDriverExtension]),
   Metric.bindTo(MultiMetric),
 
   bind(Options.CAPABILITIES)

modules/benchpress/src/web_driver_adapter.ts

@@ -16,6 +16,7 @@ export class WebDriverAdapter {
 
   waitFor(callback: Function): Promise<any> { throw new BaseException('NYI'); }
   executeScript(script: string): Promise<any> { throw new BaseException('NYI'); }
+  executeAsyncScript(script: string): Promise<any> { throw new BaseException('NYI'); }
   capabilities(): Promise<Map<string, any>> { throw new BaseException('NYI'); }
   logs(type: string): Promise<List<any>> { throw new BaseException('NYI'); }
 }

modules/benchpress/src/webdriver/async_webdriver_adapter.dart

@@ -16,6 +16,10 @@ class AsyncWebDriverAdapter extends WebDriverAdapter {
     return _driver.execute(script, const []);
   }
 
+  Future executeAsyncScript(String script) {
+    return _driver.executeAsync(script, const []);
+  }
+
   Future<Map> capabilities() {
     return new Future.value(_driver.capabilities);
   }

modules/benchpress/src/webdriver/firefox_driver_extension.ts

@@ -0,0 +1,49 @@
+import {bind, Binding} from 'angular2/di';
+import {isPresent, StringWrapper} from 'angular2/src/facade/lang';
+import {WebDriverExtension, PerfLogFeatures} from '../web_driver_extension';
+import {WebDriverAdapter} from '../web_driver_adapter';
+import {Promise} from 'angular2/src/facade/async';
+
+export class FirefoxDriverExtension extends WebDriverExtension {
+  static get BINDINGS(): List<Binding> { return _BINDINGS; }
+
+  private _profilerStarted: boolean;
+
+  constructor(private _driver: WebDriverAdapter) {
+    super();
+    this._profilerStarted = false;
+  }
+
+  gc() { return this._driver.executeScript('window.forceGC()'); }
+
+  timeBegin(name: string): Promise<any> {
+    if (!this._profilerStarted) {
+      this._profilerStarted = true;
+      this._driver.executeScript('window.startProfiler();');
+    }
+    return this._driver.executeScript('window.markStart("' + name + '");');
+  }
+
+  timeEnd(name: string, restartName: string = null): Promise<any> {
+    var script = 'window.markEnd("' + name + '");';
+    if (isPresent(restartName)) {
+      script += 'window.markStart("' + restartName + '");';
+    }
+    return this._driver.executeScript(script);
+  }
+
+  readPerfLog(): Promise<any> {
+    return this._driver.executeAsyncScript('var cb = arguments[0]; window.getProfile(cb);');
+  }
+
+  perfLogFeatures(): PerfLogFeatures { return new PerfLogFeatures({render: true, gc: true}); }
+
+  supports(capabilities: StringMap<string, any>): boolean {
+    return StringWrapper.equals(capabilities['browserName'].toLowerCase(), 'firefox');
+  }
+}
+
+var _BINDINGS = [
+  bind(FirefoxDriverExtension)
+      .toFactory((driver) => new FirefoxDriverExtension(driver), [WebDriverAdapter])
+];

modules/benchpress/src/webdriver/selenium_webdriver_adapter.ts

@@ -30,6 +30,10 @@ export class SeleniumWebDriverAdapter extends WebDriverAdapter {
     return this._convertPromise(this._driver.executeScript(script));
   }
 
+  executeAsyncScript(script: string): Promise<any> {
+    return this._convertPromise(this._driver.executeAsyncScript(script));
+  }
+
   capabilities(): Promise<any> {
     return this._convertPromise(
         this._driver.getCapabilities().then((capsObject) => capsObject.toJSON()));

modules/benchpress/test/firefox_extension/conf.ts

@@ -1,13 +1,14 @@
 /// <reference path="../../../angular2/typings/node/node.d.ts" />
+require('traceur/bin/traceur-runtime.js');
+require('reflect-metadata');
 var testHelper = require('../../src/firefox_extension/lib/test_helper.js');
 
-// Where to save profile results (parent folder must exist)
-var PROFILE_SAVE_PATH = './perfProfile.json';
-
 exports.config = {
-  specs: ['spec.js'],
+  specs: ['spec.js', 'sample_benchmark.js'],
 
-  getMultiCapabilities: function() { return testHelper.getFirefoxProfileWithExtension(); },
+  framework: 'jasmine2',
 
-  params: {profileSavePath: testHelper.getAbsolutePath(PROFILE_SAVE_PATH)}
+  jasmineNodeOpts: {showColors: true, defaultTimeoutInterval: 1200000},
+
+  getMultiCapabilities: function() { return testHelper.getFirefoxProfileWithExtension(); }
 };

modules/benchpress/test/firefox_extension/parser_util_spec.dart

@@ -0,0 +1,5 @@
+library benchpress.test.firefox_extension.parser_util_spec;
+
+main() {
+
+}

modules/benchpress/test/firefox_extension/parser_util_spec.ts

@@ -0,0 +1,102 @@
+import {convertPerfProfileToEvents} from 'benchpress/src/firefox_extension/lib/parser_util';
+
+function assertEventsEqual(actualEvents, expectedEvents) {
+  expect(actualEvents.length == expectedEvents.length);
+  for (var i = 0; i < actualEvents.length; ++i) {
+    var actualEvent = actualEvents[i];
+    var expectedEvent = expectedEvents[i];
+    for (var key in actualEvent) {
+      expect(actualEvent[key]).toEqual(expectedEvent[key]);
+    }
+  }
+};
+
+export function main() {
+  describe('convertPerfProfileToEvents', function() {
+    it('should convert single instantaneous event', function() {
+      var profileData = {
+        threads: [
+          {samples: [{time: 1, frames: [{location: 'FirefoxDriver.prototype.executeScript'}]}]}
+        ]
+      };
+      var perfEvents = convertPerfProfileToEvents(profileData);
+      assertEventsEqual(perfEvents, [{ph: 'X', ts: 1, name: 'script'}]);
+    });
+
+    it('should convert single non-instantaneous event', function() {
+      var profileData = {
+        threads: [
+          {
+            samples: [
+              {time: 1, frames: [{location: 'FirefoxDriver.prototype.executeScript'}]},
+              {time: 2, frames: [{location: 'FirefoxDriver.prototype.executeScript'}]},
+              {time: 100, frames: [{location: 'FirefoxDriver.prototype.executeScript'}]}
+            ]
+          }
+        ]
+      };
+      var perfEvents = convertPerfProfileToEvents(profileData);
+      assertEventsEqual(perfEvents,
+                        [{ph: 'B', ts: 1, name: 'script'}, {ph: 'E', ts: 100, name: 'script'}]);
+    });
+
+    it('should convert multiple instantaneous events', function() {
+      var profileData = {
+        threads: [
+          {
+            samples: [
+              {time: 1, frames: [{location: 'FirefoxDriver.prototype.executeScript'}]},
+              {time: 2, frames: [{location: 'PresShell::Paint'}]}
+            ]
+          }
+        ]
+      };
+      var perfEvents = convertPerfProfileToEvents(profileData);
+      assertEventsEqual(perfEvents,
+                        [{ph: 'X', ts: 1, name: 'script'}, {ph: 'X', ts: 2, name: 'render'}]);
+    });
+
+    it('should convert multiple mixed events', function() {
+      var profileData = {
+        threads: [
+          {
+            samples: [
+              {time: 1, frames: [{location: 'FirefoxDriver.prototype.executeScript'}]},
+              {time: 2, frames: [{location: 'PresShell::Paint'}]},
+              {time: 5, frames: [{location: 'FirefoxDriver.prototype.executeScript'}]},
+              {time: 10, frames: [{location: 'FirefoxDriver.prototype.executeScript'}]}
+            ]
+          }
+        ]
+      };
+      var perfEvents = convertPerfProfileToEvents(profileData);
+      assertEventsEqual(perfEvents, [
+        {ph: 'X', ts: 1, name: 'script'},
+        {ph: 'X', ts: 2, name: 'render'},
+        {ph: 'B', ts: 5, name: 'script'},
+        {ph: 'E', ts: 10, name: 'script'}
+      ]);
+    });
+
+    it('should add args to gc events', function() {
+      var profileData = {threads: [{samples: [{time: 1, frames: [{location: 'forceGC'}]}]}]};
+      var perfEvents = convertPerfProfileToEvents(profileData);
+      assertEventsEqual(perfEvents, [{ph: 'X', ts: 1, name: 'gc', args: {usedHeapSize: 0}}]);
+    });
+
+    it('should skip unknown events', function() {
+      var profileData = {
+        threads: [
+          {
+            samples: [
+              {time: 1, frames: [{location: 'FirefoxDriver.prototype.executeScript'}]},
+              {time: 2, frames: [{location: 'foo'}]}
+            ]
+          }
+        ]
+      };
+      var perfEvents = convertPerfProfileToEvents(profileData);
+      assertEventsEqual(perfEvents, [{ph: 'X', ts: 1, name: 'script'}]);
+    });
+  });
+};

modules/benchpress/test/firefox_extension/sample_benchmark.dart

@@ -0,0 +1,5 @@
+library benchpress.test.firefox_extension.sample_benchmark;
+
+main() {
+
+}

modules/benchpress/test/firefox_extension/sample_benchmark.ts

@@ -0,0 +1,32 @@
+var benchpress = require('../../index.js');
+var runner = new benchpress.Runner([
+  // use protractor as Webdriver client
+  benchpress.SeleniumWebDriverAdapter.PROTRACTOR_BINDINGS,
+  // use RegressionSlopeValidator to validate samples
+  benchpress.Validator.bindTo(benchpress.RegressionSlopeValidator),
+  // use 10 samples to calculate slope regression
+  benchpress.bind(benchpress.RegressionSlopeValidator.SAMPLE_SIZE).toValue(20),
+  // use the script metric to calculate slope regression
+  benchpress.bind(benchpress.RegressionSlopeValidator.METRIC).toValue('scriptTime'),
+  benchpress.bind(benchpress.Options.FORCE_GC).toValue(true)
+]);
+
+describe('deep tree baseline', function() {
+  it('should be fast!', function(done) {
+    browser.ignoreSynchronization = true;
+    browser.get('http://localhost:8001/examples/src/benchpress/');
+
+    /*
+     * Tell benchpress to click the buttons to destroy and re-create the tree for each sample.
+     * Benchpress will log the collected metrics after each sample is collected, and will stop
+     * sampling as soon as the calculated regression slope for last 20 samples is stable.
+     */
+    runner.sample({
+            id: 'baseline',
+            execute: function() { $('button')
+                                      .click(); },
+            bindings: [benchpress.bind(benchpress.Options.SAMPLE_DESCRIPTION).toValue({depth: 9})]
+          })
+        .then(done, done.fail);
+  });
+});

modules/benchpress/test/firefox_extension/spec.dart

@@ -1,2 +1,5 @@
 library benchpress.test.firefox_extension.spec;
-//no dart implementation
+
+main() {
+
+}

modules/benchpress/test/firefox_extension/spec.ts

@@ -2,26 +2,15 @@
 /// <reference path="../../../angular2/typings/angular-protractor/angular-protractor.d.ts" />
 /// <reference path="../../../angular2/typings/jasmine/jasmine.d.ts" />
 
-var fs = require('fs');
+var assertEventsContainsName = function(events, eventName) {
-
+  var found = false;
-var validateFile = function() {
+  for (var i = 0; i < events.length; ++i) {
-  try {
+    if (events[i].name == eventName) {
-    var content = fs.readFileSync(browser.params.profileSavePath, 'utf8');
+      found = true;
-    // TODO(hankduan): This check is not very useful. Ideally we want to
+      break;
-    // validate that the file contains all the events that we are looking for.
-    // Pending on data transformer.
-    expect(content).toContain('forceGC');
-    // Delete file
-    fs.unlinkSync(browser.params.profileSavePath);
-  } catch (err) {
-    if (err.code === 'ENOENT') {
-      // If files doesn't exist
-      console.error('Error: firefox extension did not save profile JSON');
-    } else {
-      console.error('Error: ' + err);
     }
-    throw err;
   }
+  expect(found).toBeTruthy();
 };
 
 describe('firefox extension', function() {
@@ -37,10 +26,10 @@ describe('firefox extension', function() {
 
     browser.executeScript('window.forceGC()');
 
-    var script = 'window.stopAndRecord("' + browser.params.profileSavePath + '")';
+    browser.executeAsyncScript('var cb = arguments[0]; window.getProfile(cb);')
-    browser.executeScript(script).then(function() { console.log('stopped measuring perf'); });
+        .then(function(profile) {
-
+          assertEventsContainsName(profile, 'gc');
-    // wait for it to finish, then validate file.
+          assertEventsContainsName(profile, 'script');
-    browser.sleep(3000).then(validateFile);
+        });
   })
 });