- + +
$SCRIPTS$ diff --git a/modules/benchmarks/src/change_detection/change_detection_benchmark.js b/modules/benchmarks/src/change_detection/change_detection_benchmark.js index c58e51ca16..6b3be5a906 100644 --- a/modules/benchmarks/src/change_detection/change_detection_benchmark.js +++ b/modules/benchmarks/src/change_detection/change_detection_benchmark.js @@ -1,7 +1,7 @@ import {ListWrapper, MapWrapper} from 'facade/collection'; import {reflector} from 'reflection/reflection'; import {isPresent} from 'facade/lang'; -import {document, DOM} from 'facade/dom'; +import {getIntParameter, bindAction} from 'e2e_test_lib/benchmark_util'; import { Lexer, @@ -12,8 +12,6 @@ import { } from 'change_detection/change_detection'; -var ITERATIONS = 500000; - class Obj { field0; field1; @@ -77,7 +75,7 @@ function setUpReflector() { }); } -function setUpBaseline() { +function setUpBaseline(iterations) { function createRow(i) { var obj = new Obj(); var index = i % 10; @@ -92,7 +90,7 @@ function setUpBaseline() { var head = createRow(0); var current = head; - for (var i = 1; i < ITERATIONS; i++) { + for (var i = 1; i < iterations; i++) { var newRow = createRow(i); current.next = newRow; current = newRow; @@ -100,7 +98,7 @@ function setUpBaseline() { return head; } -function setUpChangeDetection() { +function setUpChangeDetection(iterations) { var dispatcher = new DummyDispatcher(); var parser = new Parser(new Lexer()); @@ -139,7 +137,7 @@ function setUpChangeDetection() { proto(9) ]; - for (var i = 0; i < ITERATIONS; ++i) { + for (var i = 0; i < iterations; ++i) { var obj = new Obj(); var index = i % 10; obj.setField(index, i); @@ -154,11 +152,13 @@ function setUpChangeDetection() { } export function main () { - setUpReflector(); - var baselineHead = setUpBaseline(); - var ng2ChangeDetector = setUpChangeDetection(); + var iterations = getIntParameter('iterations'); - function baselineDetectChanges(_) { + setUpReflector(); + var baselineHead = setUpBaseline(iterations); + var ng2ChangeDetector = setUpChangeDetection(iterations); + + function baselineDetectChanges() { var current = baselineHead; while (isPresent(current)) { if (current.getter(current.obj) !== current.previousValue) { @@ -168,12 +168,12 @@ export function main () { } } - function ng2DetectChanges(_) { + function ng2DetectChanges() { ng2ChangeDetector.detectChanges(); } - DOM.on(DOM.querySelector(document, '#ng2DetectChanges'), 'click', ng2DetectChanges); - DOM.on(DOM.querySelector(document, '#baselineDetectChanges'), 'click', baselineDetectChanges); + bindAction('#ng2DetectChanges', ng2DetectChanges); + bindAction('#baselineDetectChanges', baselineDetectChanges); } diff --git a/modules/benchmarks/src/compiler/compiler_benchmark.html b/modules/benchmarks/src/compiler/compiler_benchmark.html index 460bb9bf54..30929d4478 100644 --- a/modules/benchmarks/src/compiler/compiler_benchmark.html +++ b/modules/benchmarks/src/compiler/compiler_benchmark.html @@ -2,8 +2,19 @@ - - ++ + +
+
$SCRIPTS$ diff --git a/modules/benchmarks/src/compiler/selector_benchmark.js b/modules/benchmarks/src/compiler/selector_benchmark.js index e0e6cab4de..8cbe7485db 100644 --- a/modules/benchmarks/src/compiler/selector_benchmark.js +++ b/modules/benchmarks/src/compiler/selector_benchmark.js @@ -1,46 +1,45 @@ -import {document, DOM} from 'facade/dom'; - import {SelectorMatcher} from "core/compiler/selector"; import {CssSelector} from "core/compiler/selector"; import {StringWrapper, Math} from 'facade/lang'; import {ListWrapper} from 'facade/collection'; - -var COUNT = 1000; +import {getIntParameter, bindAction} from 'e2e_test_lib/benchmark_util'; export function main() { + var count = getIntParameter('selectors'); + var fixedMatcher; var fixedSelectorStrings = []; var fixedSelectors = []; - for (var i=0; i+ + + + +
+ + +
$SCRIPTS$ diff --git a/modules/benchmarks/src/element_injector/element_injector_benchmark.js b/modules/benchmarks/src/element_injector/element_injector_benchmark.js index f35218db38..08dffa1d6b 100644 --- a/modules/benchmarks/src/element_injector/element_injector_benchmark.js +++ b/modules/benchmarks/src/element_injector/element_injector_benchmark.js @@ -1,10 +1,9 @@ import {reflector} from 'reflection/reflection'; import {Injector} from 'di/di'; import {ProtoElementInjector} from 'core/compiler/element_injector'; -import {document, DOM} from 'facade/dom'; +import {getIntParameter, bindAction} from 'e2e_test_lib/benchmark_util'; var count = 0; -var ITERATIONS = 20000; function setupReflector() { reflector.registerType(A, { @@ -25,6 +24,8 @@ function setupReflector() { } export function main() { + var iterations = getIntParameter('iterations'); + setupReflector(); var appInjector = new Injector([]); @@ -32,22 +33,22 @@ export function main() { var proto = new ProtoElementInjector(null, 0, bindings); var elementInjector = proto.instantiate(null,null); - function instantiate (_) { - for (var i = 0; i < ITERATIONS; ++i) { + function instantiate () { + for (var i = 0; i < iterations; ++i) { var ei = proto.instantiate(null, null); ei.instantiateDirectives(appInjector, null, null); } } - function instantiateDirectives (_) { - for (var i = 0; i < ITERATIONS; ++i) { + function instantiateDirectives () { + for (var i = 0; i < iterations; ++i) { elementInjector.clearDirectives(); elementInjector.instantiateDirectives(appInjector, null, null); } } - DOM.on(DOM.querySelector(document, '#instantiate'), 'click', instantiate); - DOM.on(DOM.querySelector(document, '#instantiateDirectives'), 'click', instantiateDirectives); + bindAction('#instantiate', instantiate); + bindAction('#instantiateDirectives', instantiateDirectives); } class A { diff --git a/modules/benchmarks/src/tree/tree_benchmark.html b/modules/benchmarks/src/tree/tree_benchmark.html index df33985731..5df56dcf6b 100644 --- a/modules/benchmarks/src/tree/tree_benchmark.html +++ b/modules/benchmarks/src/tree/tree_benchmark.html @@ -2,6 +2,14 @@ +diff --git a/modules/benchmarks/src/tree/tree_benchmark.js b/modules/benchmarks/src/tree/tree_benchmark.js index a79aa1e816..ae0497dadd 100644 --- a/modules/benchmarks/src/tree/tree_benchmark.js +++ b/modules/benchmarks/src/tree/tree_benchmark.js @@ -10,8 +10,7 @@ import {LifeCycle} from 'core/life_cycle/life_cycle'; import {reflector} from 'reflection/reflection'; import {DOM, document, window, Element, gc} from 'facade/dom'; import {isPresent} from 'facade/lang'; - -var MAX_DEPTH = 9; +import {getIntParameter, bindAction} from 'e2e_test_lib/benchmark_util'; function setupReflector() { // TODO: Put the general calls to reflector.register... in a shared file @@ -121,14 +120,16 @@ function setupReflector() { } export function main() { - setupReflector(); + var maxDepth = getIntParameter('depth'); + + setupReflector(); var app; var changeDetector; var baselineRootTreeComponent; var count = 0; - function ng2DestroyDom(_) { + function ng2DestroyDom() { // TODO: We need an initial value as otherwise the getter for data.value will fail // --> this should be already caught in change detection! app.initData = new TreeNode('', null, null); @@ -136,16 +137,16 @@ export function main() { } function profile(create, destroy, name) { - return function(_) { + return function() { window.console.profile(name + ' w GC'); var duration = 0; var count = 0; while(count++ < 150) { gc(); var start = window.performance.now(); - create(_); + create(); duration += window.performance.now() - start; - destroy(_); + destroy(); } window.console.profileEnd(name + ' w GC'); window.console.log(`Iterations: ${count}; time: ${duration / count} ms / iteration`); @@ -155,21 +156,21 @@ export function main() { count = 0; while(count++ < 150) { var start = window.performance.now(); - create(_); + create(); duration += window.performance.now() - start; - destroy(_); + destroy(); } window.console.profileEnd(name + ' w/o GC'); window.console.log(`Iterations: ${count}; time: ${duration / count} ms / iteration`); }; } - function ng2CreateDom(_) { + function ng2CreateDom() { var values = count++ % 2 == 0 ? ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '*'] : ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', '-']; - app.initData = buildTree(MAX_DEPTH, values, 0); + app.initData = buildTree(maxDepth, values, 0); changeDetector.detectChanges(); } @@ -179,33 +180,35 @@ export function main() { bootstrap(AppComponent).then((injector) => { changeDetector = injector.get(ChangeDetector); app = injector.get(AppComponent); - DOM.on(DOM.querySelector(document, '#ng2DestroyDom'), 'click', ng2DestroyDom); - DOM.on(DOM.querySelector(document, '#ng2CreateDom'), 'click', ng2CreateDom); - DOM.on(DOM.querySelector(document, '#ng2UpdateDomProfile'), 'click', profile(ng2CreateDom, noop, 'ng2-update')); - DOM.on(DOM.querySelector(document, '#ng2CreateDomProfile'), 'click', profile(ng2CreateDom, ng2DestroyDom, 'ng2-create')); + bindAction('#ng2DestroyDom', ng2DestroyDom); + bindAction('#ng2CreateDom', ng2CreateDom); + bindAction('#ng2UpdateDomProfile', profile(ng2CreateDom, noop, 'ng2-update')); + bindAction('#ng2CreateDomProfile', profile(ng2CreateDom, ng2DestroyDom, 'ng2-create')); }); } - function baselineDestroyDom(_) { + function baselineDestroyDom() { baselineRootTreeComponent.update(new TreeNode('', null, null)); } - function baselineCreateDom(_) { + function baselineCreateDom() { var values = count++ % 2 == 0 ? ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '*'] : ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', '-']; - baselineRootTreeComponent.update(buildTree(MAX_DEPTH, values, 0)); + baselineRootTreeComponent.update(buildTree(maxDepth, values, 0)); } function initBaseline() { var tree = DOM.createElement('tree'); DOM.appendChild(DOM.querySelector(document, 'baseline'), tree); baselineRootTreeComponent = new BaseLineTreeComponent(tree); - DOM.on(DOM.querySelector(document, '#baselineDestroyDom'), 'click', baselineDestroyDom); - DOM.on(DOM.querySelector(document, '#baselineCreateDom'), 'click', baselineCreateDom); - DOM.on(DOM.querySelector(document, '#baselineUpdateDomProfile'), 'click', profile(baselineCreateDom, noop, 'baseline-update')); - DOM.on(DOM.querySelector(document, '#baselineCreateDomProfile'), 'click', profile(baselineCreateDom, baselineDestroyDom, 'baseline-create')); + + bindAction('#baselineDestroyDom', baselineDestroyDom); + bindAction('#baselineCreateDom', baselineCreateDom); + + bindAction('#baselineUpdateDomProfile', profile(baselineCreateDom, noop, 'baseline-update')); + bindAction('#baselineCreateDomProfile', profile(baselineCreateDom, baselineDestroyDom, 'baseline-create')); } initNg2(); diff --git a/modules/benchmarks_external/e2e_test/compiler_perf.es6 b/modules/benchmarks_external/e2e_test/compiler_perf.es6 index 891a395389..d6cbe89613 100644 --- a/modules/benchmarks_external/e2e_test/compiler_perf.es6 +++ b/modules/benchmarks_external/e2e_test/compiler_perf.es6 @@ -1,39 +1,31 @@ -"use strict"; -var benchpress = require('../../../tools/benchpress/index.js'); +var perfUtil = require('../../e2e_test_lib/e2e_test/perf_util'); describe('ng1.x compiler benchmark', function () { var URL = 'benchmarks_external/web/compiler/compiler_benchmark.html'; - afterEach(benchpress.verifyNoBrowserErrors); + afterEach(perfUtil.verifyNoBrowserErrors); it('should log withBinding stats', function() { - browser.get(URL); - runClickBenchmark({ + perfUtil.runClickBenchmark({ + url: URL, buttons: ['#compileWithBindings'], - logId: 'ng1.compile.withBindings' + id: 'ng1.compile.withBindings', + params: [{ + name: 'elementCount', value: 150 + }] }); }); it('should log noBindings stats', function() { - browser.get(URL); - runClickBenchmark({ + perfUtil.runClickBenchmark({ + url: URL, buttons: ['#compileNoBindings'], - logId: 'ng1.compile.noBindings' + id: 'ng1.compile.noBindings', + params: [{ + name: 'elementCount', value: 150 + }] }); }); }); - -function runClickBenchmark(config) { - var buttons = config.buttons.map(function(selector) { - return $(selector); - }); - var params = Object.create(browser.params.benchmark); - params.logId = browser.params.lang+'.'+config.logId; - benchpress.runBenchmark(params, function() { - buttons.forEach(function(button) { - button.click(); - }); - }); -} diff --git a/modules/benchmarks_external/e2e_test/compiler_spec.es6 b/modules/benchmarks_external/e2e_test/compiler_spec.es6 index 5fb262d68b..183ebec227 100644 --- a/modules/benchmarks_external/e2e_test/compiler_spec.es6 +++ b/modules/benchmarks_external/e2e_test/compiler_spec.es6 @@ -1,20 +1,14 @@ -var benchpress = require('../../../tools/benchpress/index.js'); +var testUtil = require('../../e2e_test_lib/e2e_test/test_util'); describe('ng1.x compiler benchmark', function () { var URL = 'benchmarks_external/web/compiler/compiler_benchmark.html'; - afterEach(benchpress.verifyNoBrowserErrors); + afterEach(testUtil.verifyNoBrowserErrors); it('should not throw errors', function() { browser.get(URL); - clickAll(['#compileWithBindings', '#compileNoBindings']); + testUtil.clickAll(['#compileWithBindings', '#compileNoBindings']); }); }); - -function clickAll(buttonSelectors) { - buttonSelectors.forEach(function(selector) { - $(selector).click(); - }); -} diff --git a/modules/benchmarks_external/e2e_test/tree_perf.es6 b/modules/benchmarks_external/e2e_test/tree_perf.es6 index baaab54888..90058fc805 100644 --- a/modules/benchmarks_external/e2e_test/tree_perf.es6 +++ b/modules/benchmarks_external/e2e_test/tree_perf.es6 @@ -1,31 +1,20 @@ -"use strict"; -var benchpress = require('../../../tools/benchpress/index.js'); +var perfUtil = require('../../e2e_test_lib/e2e_test/perf_util'); describe('ng1.x tree benchmark', function () { var URL = 'benchmarks_external/web/tree/tree_benchmark.html'; - afterEach(benchpress.verifyNoBrowserErrors); + afterEach(perfUtil.verifyNoBrowserErrors); it('should log the stats', function() { - browser.get(URL); - runClickBenchmark({ + perfUtil.runClickBenchmark({ + url: URL, buttons: ['#destroyDom', '#createDom'], - logId: 'ng1.tree' + id: 'ng1.tree', + params: [{ + name: 'depth', value: 9 + }] }); }); }); - -function runClickBenchmark(config) { - var buttons = config.buttons.map(function(selector) { - return $(selector); - }); - var params = Object.create(browser.params.benchmark); - params.logId = browser.params.lang+'.'+config.logId; - benchpress.runBenchmark(params, function() { - buttons.forEach(function(button) { - button.click(); - }); - }); -} diff --git a/modules/benchmarks_external/e2e_test/tree_spec.es6 b/modules/benchmarks_external/e2e_test/tree_spec.es6 index 19f6bbbbc5..83202bc27d 100644 --- a/modules/benchmarks_external/e2e_test/tree_spec.es6 +++ b/modules/benchmarks_external/e2e_test/tree_spec.es6 @@ -1,20 +1,14 @@ -var benchpress = require('../../../tools/benchpress/index.js'); +var testUtil = require('../../e2e_test_lib/e2e_test/test_util'); describe('ng1.x tree benchmark', function () { var URL = 'benchmarks_external/web/tree/tree_benchmark.html'; - afterEach(benchpress.verifyNoBrowserErrors); + afterEach(testUtil.verifyNoBrowserErrors); it('should not throw errors', function() { browser.get(URL); - clickAll(['#createDom', '#destroyDom']); + testUtil.clickAll(['#createDom', '#destroyDom']); }); }); - -function clickAll(buttonSelectors) { - buttonSelectors.forEach(function(selector) { - $(selector).click(); - }); -} diff --git a/modules/benchmarks_external/pubspec.yaml b/modules/benchmarks_external/pubspec.yaml index 132b08601b..10c59f2752 100644 --- a/modules/benchmarks_external/pubspec.yaml +++ b/modules/benchmarks_external/pubspec.yaml @@ -2,6 +2,8 @@ name: benchmarks_external environment: sdk: '>=1.4.0' dependencies: + e2e_test_lib: + path: ../e2e_test_lib angular: ">=1.0.0 <2.0.0" browser: '>=0.10.0 <0.11.0' transformers: diff --git a/modules/benchmarks_external/src/compiler/compiler_benchmark.dart b/modules/benchmarks_external/src/compiler/compiler_benchmark.dart index db03be93c9..855c7cfc1f 100644 --- a/modules/benchmarks_external/src/compiler/compiler_benchmark.dart +++ b/modules/benchmarks_external/src/compiler/compiler_benchmark.dart @@ -4,11 +4,12 @@ library compiler_benchmark_ng10; import 'package:angular/angular.dart'; import 'package:angular/application_factory.dart'; import 'dart:html'; - -var COUNT = 30; +import 'package:e2e_test_lib/benchmark_util.dart'; main() { + var count = getIntParameter('elements'); + var m = new Module() ..bind(Dir0) ..bind(Dir1) @@ -16,25 +17,25 @@ main() { ..bind(Dir3) ..bind(Dir4); - var templateWithBindings = loadTemplate('templateWithBindings', COUNT); - var templateNoBindings = loadTemplate('templateWithBindings', COUNT); + var templateWithBindings = loadTemplate('templateWithBindings', count); + var templateNoBindings = loadTemplate('templateWithBindings', count); final injector = applicationFactory().addModule(m).run(); final compiler = injector.get(Compiler); final directiveMap = injector.get(DirectiveMap); - compileWithBindings(_) { + compileWithBindings() { final cloned = templateWithBindings.clone(true); compiler([cloned], directiveMap); } - compileNoBindings(_) { + compileNoBindings() { final cloned = templateNoBindings.clone(true); compiler([cloned], directiveMap); } - document.querySelector('#compileWithBindings').addEventListener('click', compileWithBindings); - document.querySelector('#compileNoBindings').addEventListener('click', compileNoBindings); + bindAction('#compileWithBindings', compileWithBindings); + bindAction('#compileNoBindings', compileNoBindings); } diff --git a/modules/benchmarks_external/src/compiler/compiler_benchmark.es6 b/modules/benchmarks_external/src/compiler/compiler_benchmark.es6 index 15ce5ee3f0..76e3e83bc8 100644 --- a/modules/benchmarks_external/src/compiler/compiler_benchmark.es6 +++ b/modules/benchmarks_external/src/compiler/compiler_benchmark.es6 @@ -1,5 +1,5 @@ // compiler benchmark in AngularJS 1.x -var COUNT = 30; +import {getIntParameter, bindAction} from 'e2e_test_lib/benchmark_util'; export function main() { var ngEl = document.createElement('div'); @@ -74,19 +74,20 @@ angular.module('app', []) }; }]) .run(['$compile', function($compile) { - var templateNoBindings = loadTemplate('templateNoBindings', COUNT); - var templateWithBindings = loadTemplate('templateWithBindings', COUNT); + var count = getIntParameter('elements'); + var templateNoBindings = loadTemplate('templateNoBindings', count); + var templateWithBindings = loadTemplate('templateWithBindings', count); - document.querySelector('#compileWithBindings').addEventListener('click', compileWithBindings, false); - document.querySelector('#compileNoBindings').addEventListener('click', compileNoBindings, false); + bindAction('#compileWithBindings', compileWithBindings); + bindAction('#compileNoBindings', compileNoBindings); - function compileNoBindings(_) { + function compileNoBindings() { // Need to clone every time as the compiler might modify the template! var cloned = templateNoBindings.cloneNode(true); $compile(cloned); } - function compileWithBindings(_) { + function compileWithBindings() { // Need to clone every time as the compiler might modify the template! var cloned = templateWithBindings.cloneNode(true); $compile(cloned); diff --git a/modules/benchmarks_external/src/compiler/compiler_benchmark.html b/modules/benchmarks_external/src/compiler/compiler_benchmark.html index 460bb9bf54..30929d4478 100644 --- a/modules/benchmarks_external/src/compiler/compiler_benchmark.html +++ b/modules/benchmarks_external/src/compiler/compiler_benchmark.html @@ -2,8 +2,19 @@
- - ++ + +
diff --git a/modules/e2e_test_lib/e2e_test/perf_util.es6 b/modules/e2e_test_lib/e2e_test/perf_util.es6 new file mode 100644 index 0000000000..1136ebe367 --- /dev/null +++ b/modules/e2e_test_lib/e2e_test/perf_util.es6 @@ -0,0 +1,25 @@ +var benchpress = require('../../../tools/benchpress/index.js'); +var webdriver = require('protractor/node_modules/selenium-webdriver'); + +module.exports = { + runClickBenchmark: runClickBenchmark, + verifyNoBrowserErrors: benchpress.verifyNoBrowserErrors +}; + +function runClickBenchmark(config) { + var url = encodeURI(config.url + '?' + config.params.map(function(param) { + return param.name + '=' + param.value; + }).join('&')); + browser.get(url); + var buttons = config.buttons.map(function(selector) { + return $(selector); + }); + var benchmarkConfig = Object.create(browser.params.benchmark); + benchmarkConfig.id = browser.params.lang+'.'+config.id; + benchmarkConfig.params = config.params; + benchpress.runBenchmark(benchmarkConfig, function() { + buttons.forEach(function(button) { + button.click(); + }); + }); +} \ No newline at end of file diff --git a/modules/e2e_test_lib/e2e_test/test_util.es6 b/modules/e2e_test_lib/e2e_test/test_util.es6 new file mode 100644 index 0000000000..bfc829f492 --- /dev/null +++ b/modules/e2e_test_lib/e2e_test/test_util.es6 @@ -0,0 +1,12 @@ +var benchpress = require('../../../tools/benchpress/index.js'); + +module.exports = { + verifyNoBrowserErrors: benchpress.verifyNoBrowserErrors, + clickAll: clickAll +}; + +function clickAll(buttonSelectors) { + buttonSelectors.forEach(function(selector) { + $(selector).click(); + }); +} diff --git a/modules/e2e_test_lib/pubspec.yaml b/modules/e2e_test_lib/pubspec.yaml new file mode 100644 index 0000000000..bf9a3f809b --- /dev/null +++ b/modules/e2e_test_lib/pubspec.yaml @@ -0,0 +1,7 @@ +name: e2e_test_lib +environment: + sdk: '>=1.4.0' +dependencies: + facade: + path: ../facade + browser: '>=0.10.0 <0.11.0' diff --git a/modules/e2e_test_lib/src/benchmark_util.js b/modules/e2e_test_lib/src/benchmark_util.js new file mode 100644 index 0000000000..a82c4f0e2c --- /dev/null +++ b/modules/e2e_test_lib/src/benchmark_util.js @@ -0,0 +1,17 @@ +import {DOM, document, location} from 'facade/dom'; +import {NumberWrapper, BaseException, isBlank} from 'facade/lang'; + +export function getIntParameter(name:string) { + var el = DOM.querySelector(document, `input[name="${name}"]`); + if (isBlank(el)) { + throw new BaseException(`Could not find and input field with name ${name}`); + } + return NumberWrapper.parseInt(el.value, 10); +} + +export function bindAction(selector:string, callback:Function) { + var el = DOM.querySelector(document, selector); + DOM.on(el, 'click', function(_) { + callback(); + }); +} \ No newline at end of file diff --git a/package.json b/package.json index 402c67239f..3f17d1e2c0 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,9 @@ "systemjs": "^0.9.1", "traceur": "vojtajina/traceur-compiler#disable-getters-setters", "which": "~1", - "zone.js": "0.3.0" + "zone.js": "0.3.0", + "googleapis": "1.0.x", + "node-uuid": "1.4.x" }, "devDependencies": { "bower": "^1.3.12", diff --git a/perf-cloud-secret.template.js b/perf-cloud-secret.template.js new file mode 100644 index 0000000000..d4d2f27b34 --- /dev/null +++ b/perf-cloud-secret.template.js @@ -0,0 +1,11 @@ +module.exports = { + auth: { + "private_key_id": "1234", + "private_key": "-----BEGIN PRIVATE KEY-----SOME PRIVATE KEY-----END PRIVATE KEY-----\n", + "client_email": "SOME_EMAIL@developer.gserviceaccount.com", + "client_id": "SOME_ID", + "type": "service_account" + }, + projectId: 'angular-perf', + datasetId: 'benchmarks' +}; \ No newline at end of file diff --git a/protractor-perf-shared.js b/protractor-perf-shared.js index a2751f0567..d9f7418d28 100644 --- a/protractor-perf-shared.js +++ b/protractor-perf-shared.js @@ -1,6 +1,13 @@ // load traceur runtime as our tests are written in es6 require('traceur/bin/traceur-runtime.js'); +var cloudReporterConfig; +try { + cloudReporterConfig = require('./perf-cloud-secret.js'); +} catch (e) { + cloudReporterConfig = null; +} + var config = exports.config = { specs: ['dist/cjs/**/*_perf.js'], @@ -8,16 +15,16 @@ var config = exports.config = { params: { benchmark: { // size of the sample to take - sampleSize: 10, - targetCoefficientOfVariation: 4, + sampleSize: 20, timeout: 20000, metrics: ['script', 'render', 'gcAmount', 'gcAmountInScript', 'gcTime'], - // run mode of the benchmark: - // - detect: auto detect whether to force gc - // - forceGc: forces a gc before every run and ignores no runs - // - noGcInScript: ignore runs that have gc while a script was executing - // - plain: does not force nor ignore runs - mode: 'detect' + // forces a gc after every run + forceGc: false, + reporters: [ + require('./dist/cjs/tools/benchpress/src/console_reporter.js'), + cloudReporterConfig ? require('./dist/cjs/tools/benchpress/src/cloud_reporter.js') : null, + ], + cloudReporter: cloudReporterConfig } }, diff --git a/tools/benchpress/src/benchmark.es6 b/tools/benchpress/src/benchmark.es6 index 7dd9c73320..1d62646f51 100644 --- a/tools/benchpress/src/benchmark.es6 +++ b/tools/benchpress/src/benchmark.es6 @@ -1,6 +1,7 @@ -var stats = require('./stats'); -var reporter = require('./reporter'); +var statistics = require('./statistics'); var commands = require('./commands'); +var nodeUuid = require('node-uuid'); +var webdriver = require('protractor/node_modules/selenium-webdriver'); var SUPPORTED_METRICS = { script: true, @@ -12,52 +13,6 @@ var SUPPORTED_METRICS = { render: true }; -var RUN_MODE = { - detect: function(prevState, benchmarkData, iterationIndex) { - var gcInScriptCount = prevState.gcInScriptCount || 0; - if (benchmarkData.gcAmountInScript) { - gcInScriptCount++; - } - var ignoreRun = !!benchmarkData.gcAmountInScript; - var nextMode = RUN_MODE.detect; - if (iterationIndex > 10) { - if (gcInScriptCount / iterationIndex > 0.7) { - nextMode = RUN_MODE.forceGc; - } else { - nextMode = RUN_MODE.noGcInScript; - } - } - return { - forceGc: false, - ignoreRun: ignoreRun, - gcInScriptCount: gcInScriptCount, - nextMode: nextMode - }; - }, - forceGc: function() { - return { - forceGc: true, - ignoreRun: false, - nextMode: RUN_MODE.forceGc - } - }, - noGcInScript: function(prevState, benchmarkData) { - var ignoreRun = !!benchmarkData.gcAmountInScript; - return { - forceGc: false, - ignoreRun: ignoreRun, - nextMode: RUN_MODE.noGcInScript - } - }, - plain: function() { - return { - forceGc: false, - ignoreRun: false, - nextMode: RUN_MODE.plain - } - } -}; - var nextTimestampId = 0; module.exports = { @@ -66,133 +21,139 @@ module.exports = { }; function runBenchmark(config, workCallback) { - config.metrics.forEach(function(metric) { + var sampleId = nodeUuid.v1(); + var reporters = config.reporters.map(function(Class) { + return new Class(sampleId, config); + }); + var scriptMetricIndex = -1; + config.metrics.forEach(function(metric, index) { if (!(metric in SUPPORTED_METRICS)) { throw new Error('Metric '+metric+' is not suported by benchpress right now'); } + if (metric === 'script') { + scriptMetricIndex = index; + } }); - var ROW_FORMAT = ['%-40s', '%12s'].concat(config.metrics.map(function() { - return '%12s'; - })).join(' | '); - - var benchmarkStatsAggregator = stats.createObjectStatsAggregator(config.metrics, config.sampleSize); + if (scriptMetricIndex === -1) { + throw new Error('Metric "script" needs to be included in the metrics'); + } var startTime = Date.now(); - startLoop().then(endLoop); - - function startLoop(gcData) { - reporter.printHeading('SCRIPT DATA: sampling size '+config.sampleSize); - reporter.printTableHeader(ROW_FORMAT, ['name', 'action'].concat(config.metrics)); - if (!(config.mode in RUN_MODE)) { - throw new Error('Unknown mode '+config.mode); - } - return loop(0, { - forceGc: false, - ignoreRun: false, - nextMode: RUN_MODE[config.mode] - }); - } - - function endLoop(stats) { - reporter.printTableFooter(ROW_FORMAT, [config.logId, ''] - .concat(formatObjectStats(stats, config.metrics)) - ); - return config.metrics.map(function(metric) { - return stats[metric]; - }); - } - - function loop(iterationIndex, modeState) { - return measureTime(function() { + commands.gc(); + reporters.forEach(function(reporter) { + reporter.begin(); + }); + return measureLoop({ + index: 0, + prevSample: [], + endAfterRun: false, + work: function() { workCallback(); - if (modeState.forceGc) { - // For fast tests that don't create a lot of garbage, - // we don't want to force gc before every run as that - // can slow down the script execution time (even when we subtract - // the gc time)! - // Note: we need to call gc AFTER the actual test so the - // gc amount is added to the current test run! + if (this.endAfterRun || config.forceGc) { commands.gc(); } - }).then(function(benchmarkData) { - modeState = modeState.nextMode(modeState, benchmarkData, iterationIndex); - var action = ''; - if (modeState.ignoreRun) { - action = 'ignore'; - } else if (modeState.forceGc) { - action = 'forceGc'; + }, + process: function(data) { + var measuredValues = config.metrics.map(function(metric) { + return data.stats[metric]; + }); + var reporterData = { + values: measuredValues, + index: this.index, + records: data.records, + forceGc: this.endAfterRun || config.forceGc + }; + reporters.forEach(function(reporter) { + reporter.add(reporterData); + }); + + var newSample = this.prevSample.concat([reporterData]); + if (newSample.length > config.sampleSize) { + newSample = newSample.slice(newSample.length - config.sampleSize); } - reporter.printRow(ROW_FORMAT, [config.logId + '#' + iterationIndex, action] - .concat(formatObjectData(benchmarkData, config.metrics)) + + var result = null; + var xValues = []; + var yValues = []; + newSample.forEach(function(data, index) { + // For now, we only use the array index as x value. + // TODO(tbosch): think about whether we should use time here instead + xValues.push(index); + yValues.push(data.values[scriptMetricIndex]); + }); + var regressionSlope = statistics.getRegressionSlope( + xValues, statistics.calculateMean(xValues), + yValues, statistics.calculateMean(yValues) ); + // TODO(tbosch): ask someone who really understands statistics whether this is reasonable + // When we detect that we are not getting slower any more, + // we do one more round where we force gc so we get all the gc data before we stop. + var endAfterNextRun = ((Date.now() - startTime > config.timeout) || + (newSample.length === config.sampleSize && regressionSlope >= 0)); + return { + index: this.index+1, + work: this.work, + process: this.process, + endAfterRun: endAfterNextRun, + result: this.endAfterRun ? newSample : null, + prevSample: newSample + }; + } + }).then(function(stableSample) { + reporters.forEach(function(reporter) { + reporter.end(stableSample); + }); + }); +} - var benchmarkStats; - if (modeState.ignoreRun) { - benchmarkStats = benchmarkStatsAggregator.current; +function measureLoop(startState) { + var startTimestampId = (nextTimestampId++).toString(); + commands.timelineTimestamp(startTimestampId); + + return next(startTimestampId, startState, []); + + function next(startTimestampId, state, lastRecords) { + state.work(); + var endTimestampId = (nextTimestampId++).toString(); + commands.timelineTimestamp(endTimestampId); + + return readStats(startTimestampId, endTimestampId, lastRecords).then(function(data) { + var nextState = state.process({ + stats: data.stats, + records: data.records + }); + if (nextState.result) { + return nextState.result; } else { - benchmarkStats = benchmarkStatsAggregator(benchmarkData); + return next(endTimestampId, nextState, data.lastRecords); } - - if (Date.now() - startTime > config.timeout) { - return benchmarkStats; - } - if (benchmarkStats && - ( - benchmarkStats.script.count >= config.sampleSize && - benchmarkStats.script.coefficientOfVariation < config.targetCoefficientOfVariation) - ) { - return benchmarkStats - } - return loop(iterationIndex+1, modeState); }); } -} -function formatObjectData(data, props) { - return props.map(function(prop) { - var val = data[prop]; - if (typeof val === 'number') { - return val.toFixed(2); - } else { - return val; - } - }); -} - -function formatObjectStats(stats, props) { - return props.map(function(prop) { - var entry = stats[prop]; - return entry.mean.toFixed(2) + '\u00B1' + entry.coefficientOfVariation.toFixed(0)+ '%'; - }); -} - -function measureTime(callback) { - var startId = (nextTimestampId++).toString(); - var endId = (nextTimestampId++).toString(); - commands.timelineTimestamp(startId); - callback(); - commands.timelineTimestamp(endId); - var allRecords = []; - return readResult(); - - function readResult() { - return commands.timelineRecords().then(function(records) { - allRecords.push.apply(allRecords, records); - var stats = sumTimelineRecords(allRecords, startId, endId); - if (stats.timeStamps.indexOf(startId) === -1 || - stats.timeStamps.indexOf(endId) === -1) { + function readStats(startTimestampId, endTimestampId, lastRecords) { + return commands.timelineRecords().then(function(newRecords) { + var records = lastRecords.concat(newRecords); + var stats = sumTimelineRecords(records, startTimestampId, endTimestampId); + if (stats.timeStamps.indexOf(startTimestampId) === -1 || + stats.timeStamps.indexOf(endTimestampId) === -1) { // Sometimes the logs have not yet arrived at the webdriver - // server from the browser. + // server from the browser, so we need to wait + // TODO(tbosch): This seems to be a bug in chrome / chromedriver! // And sometimes, just waiting is not enough, so we // execute a dummy js function :-( browser.executeScript('1+1'); browser.sleep(100); - return readResult(); + return readStats(startTimestampId, endTimestampId, records); } else { - return stats; + return { + stats: stats, + records: records, + lastRecords: newRecords + }; } }); } + } function sumTimelineRecords(records, startTimeStampId, endTimeStampId) { diff --git a/tools/benchpress/src/cloud_reporter.es6 b/tools/benchpress/src/cloud_reporter.es6 new file mode 100644 index 0000000000..9b2263c71f --- /dev/null +++ b/tools/benchpress/src/cloud_reporter.es6 @@ -0,0 +1,224 @@ +var google = require('googleapis'); +var bigquery = google.bigquery('v2'); +var webdriver = require('protractor/node_modules/selenium-webdriver'); + +var HEADER_FIELDS = [ + { + "name": 'runId', + "type": 'STRING', + "description": 'uuid for the benchmark run' + }, + { + "name": 'index', + "type": 'INTEGER', + "description": 'index within the sample' + }, + { + "name": 'creationTime', + "type": 'TIMESTAMP' + }, + { + "name": 'browser', + "type": 'STRING', + "description": 'navigator.platform' + }, + { + "name": 'forceGc', + "type": 'BOOLEAN', + "description": 'whether gc was forced at end of action' + } +]; + +class CloudReporter { + constructor(runId, benchmarkConfig) { + this.stableRowsTableConfig = createTableConfig(benchmarkConfig, '_stable'); + this.allRowsTableConfig = createTableConfig(benchmarkConfig, '_all') + this.authConfig = benchmarkConfig.cloudReporter.auth; + this.benchmarkConfig = benchmarkConfig; + this.runId = runId; + this.allRows = []; + var self = this; + browser.executeScript('return navigator.userAgent').then(function(userAgent) { + self.browserUserAgent = userAgent; + }); + + } + begin() { + var self = this; + var flow = browser.driver.controlFlow(); + flow.execute(function() { + return authenticate(self.authConfig).then(function(authClient) { + self.authClient = authClient; + }); + }); + flow.execute(function() { + return webdriver.promise.all([ + getOrCreateTable(self.authClient, self.allRowsTableConfig), + getOrCreateTable(self.authClient, self.stableRowsTableConfig) + ]); + }); + } + add(data) { + this.allRows.push(this._convertToTableRow(data)); + } + end(stableSample) { + var self = this; + var flow = browser.driver.controlFlow(); + var stableRows = stableSample.map(function(data) { + return self._convertToTableRow(data); + }); + flow.execute(function() { + return webdriver.promise.all([ + insertRows(self.authClient, self.stableRowsTableConfig, stableRows), + insertRows(self.authClient, self.allRowsTableConfig, self.allRows) + ]); + }); + } + _convertToTableRow(benchpressRow) { + var tableRow = { + runId: this.runId, + index: benchpressRow.index, + creationTime: new Date(), + browser: this.browserUserAgent, + forceGc: benchpressRow.forceGc + }; + this.benchmarkConfig.params.forEach(function(param) { + tableRow['p_'+param.name] = param.value; + }); + this.benchmarkConfig.metrics.forEach(function(metric, index) { + tableRow['m_'+metric] = benchpressRow.values[index]; + }); + return tableRow; + } +} + +function createTableConfig(benchmarkConfig, tableSuffix) { + var tableId = (benchmarkConfig.id+tableSuffix).replace(/\./g, '_'); + return { + projectId: benchmarkConfig.cloudReporter.projectId, + datasetId: benchmarkConfig.cloudReporter.datasetId, + table: { + id: tableId, + fields: HEADER_FIELDS + .concat(benchmarkConfig.params.map(function(param) { + return { + "name": 'p_'+param.name, + "type": 'FLOAT' + }; + })) + .concat(benchmarkConfig.metrics.map(function(metricName) { + return { + "name": 'm_'+metricName, + "type": 'FLOAT' + }; + })) + } + }; +} + +function getOrCreateTable(authClient, tableConfig) { + return getTable(authClient, tableConfig).then(null, function(err) { + // create the table if it does not exist + return createTable(authClient, tableConfig); + }); +} + +function authenticate(authConfig) { + var authClient = new google.auth.JWT( + authConfig['client_email'], + null, + authConfig['private_key'], + ['https://www.googleapis.com/auth/bigquery'], + // User to impersonate (leave empty if no impersonation needed) + null); + + var defer = webdriver.promise.defer(); + authClient.authorize(makeNodeJsResolver(defer)); + return defer.promise.then(function() { + return authClient; + }); +} + +function getTable(authClient, tableConfig) { + // see https://cloud.google.com/bigquery/docs/reference/v2/tables/get + var params = { + auth: authClient, + projectId: tableConfig.projectId, + datasetId: tableConfig.datasetId, + tableId: tableConfig.table.id + }; + var defer = webdriver.promise.defer(); + bigquery.tables.get(params, makeNodeJsResolver(defer)); + return defer.promise; +} + +function createTable(authClient, tableConfig) { + // see https://cloud.google.com/bigquery/docs/reference/v2/tables + // see https://cloud.google.com/bigquery/docs/reference/v2/tables#resource + var params = { + auth: authClient, + projectId: tableConfig.projectId, + datasetId: tableConfig.datasetId, + resource: { + "kind": "bigquery#table", + "tableReference": { + projectId: tableConfig.projectId, + datasetId: tableConfig.datasetId, + tableId: tableConfig.table.id + }, + "schema": { + "fields": tableConfig.table.fields + } + } + }; + var defer = webdriver.promise.defer(); + bigquery.tables.insert(params, makeNodeJsResolver(defer)); + return defer.promise; +} + +function insertRows(authClient, tableConfig, rows) { + // see https://cloud.google.com/bigquery/docs/reference/v2/tabledata/insertAll + var params = { + auth: authClient, + projectId: tableConfig.projectId, + datasetId: tableConfig.datasetId, + tableId: tableConfig.table.id, + resource: { + "kind": "bigquery#tableDataInsertAllRequest", + "rows": rows.map(function(row) { + return { + json: row + } + }) + } + }; + var defer = webdriver.promise.defer(); + bigquery.tabledata.insertAll(params, makeNodeJsResolver(defer)); + return defer.promise.then(function(result) { + if (result.insertErrors) { + throw result.insertErrors.map(function(err) { + return err.errors.map(function(err) { + return err.message; + }).join('\n'); + }).join('\n'); + } + }); +} + +function makeNodeJsResolver(defer) { + return function(err, result) { + if (err) { + // Normalize errors messages from BigCloud so that they show up nicely + if (err.errors) { + err = err.errors.map(function(err) { + return err.message; + }).join('\n'); + } + defer.reject(err); + } else { + defer.fulfill(result); + } + } +} + +module.exports = CloudReporter; \ No newline at end of file diff --git a/tools/benchpress/src/console_reporter.es6 b/tools/benchpress/src/console_reporter.es6 new file mode 100644 index 0000000000..e878b27615 --- /dev/null +++ b/tools/benchpress/src/console_reporter.es6 @@ -0,0 +1,77 @@ +var vsprintf = require("sprintf-js").vsprintf; +var statistics = require("./statistics"); + +var HEADER_SEPARATORS = ['----', '----', '----', '----', '----', '----', '----']; +var FOOTER_SEPARATORS = ['====', '====', '====', '====', '====', '====', '====']; + +class ConsoleReporter { + constructor(runId, config) { + this.config = config; + this.runId = runId; + this.rowFormat = ['%12s'].concat(config.metrics.map(function() { + return '%12s'; + })).join(' | '); + } + begin() { + printHeading('BENCHMARK '+this.config.id); + console.log('sample size', this.config.sampleSize); + console.log('run id', this.runId); + console.log('params', JSON.stringify(this.config.params, null, ' ')); + printTableHeader(this.rowFormat, ['index', 'forceGc'].concat(this.config.metrics)); + } + add(data) { + var values = data.values; + var index = data.index; + printRow(this.rowFormat, ['#' + index, data.forceGc] + .concat(formatValues(values)) + ); + } + end(stableSample) { + printTableFooter(this.rowFormat, [this.config.id, ''] + .concat(formatSample(stableSample, this.config.metrics))); + } +} + +function formatValues(values) { + return values.map(function(val) { + if (typeof val === 'number') { + return val.toFixed(2); + } else { + return val; + } + }); +} + +function formatSample(sample, metrics) { + return metrics.map(function(_, metricIndex) { + var metricSample = sample.map(function(row) { + return row.values[metricIndex]; + }); + var mean = statistics.calculateMean(metricSample); + var coefficientOfVariation = statistics.calculateCoefficientOfVariation(metricSample, mean); + return mean.toFixed(2) + '\u00B1' + coefficientOfVariation.toFixed(0)+ '%'; + }); +} + +function printHeading(title) { + console.log('\n'); + console.log('## '+title); +} + +function printTableHeader(format, values) { + printRow(format, values); + // TODO(tbosch): generate separators dynamically based on the format! + printRow(format, HEADER_SEPARATORS); +} + +function printTableFooter(format, values) { + // TODO(tbosch): generate separators dynamically based on the format! + printRow(format, FOOTER_SEPARATORS); + printRow(format, values); +} + +function printRow(format, values) { + console.log(vsprintf(format, values)); +} + +module.exports = ConsoleReporter; diff --git a/tools/benchpress/src/reporter.es6 b/tools/benchpress/src/reporter.es6 deleted file mode 100644 index 9b870df824..0000000000 --- a/tools/benchpress/src/reporter.es6 +++ /dev/null @@ -1,32 +0,0 @@ -var vsprintf = require("sprintf-js").vsprintf; - -var HEADER_SEPARATORS = ['----', '----', '----', '----', '----', '----', '----']; -var FOOTER_SEPARATORS = ['====', '====', '====', '====', '====', '====', '====']; - -module.exports = { - printHeading: printHeading, - printTableHeader: printTableHeader, - printTableFooter: printTableFooter, - printRow: printRow -}; - -function printHeading(title) { - console.log('\n'); - console.log('## '+title); -} - -function printTableHeader(format, values) { - printRow(format, values); - // TODO(tbosch): generate separators dynamically based on the format! - printRow(format, HEADER_SEPARATORS); -} - -function printTableFooter(format, values) { - // TODO(tbosch): generate separators dynamically based on the format! - printRow(format, FOOTER_SEPARATORS); - printRow(format, values); -} - -function printRow(format, values) { - console.log(vsprintf(format, values)); -} \ No newline at end of file diff --git a/tools/benchpress/src/statistics.es6 b/tools/benchpress/src/statistics.es6 new file mode 100644 index 0000000000..bceba78a97 --- /dev/null +++ b/tools/benchpress/src/statistics.es6 @@ -0,0 +1,37 @@ +module.exports = { + calculateCoefficientOfVariation: calculateCoefficientOfVariation, + calculateMean: calculateMean, + calculateStandardDeviation: calculateStandardDeviation, + getRegressionSlope: getRegressionSlope +}; + +function calculateCoefficientOfVariation(sample, mean) { + return calculateStandardDeviation(sample, mean) / mean * 100; +} + +function calculateMean(sample) { + var total = 0; + sample.forEach(function(x) { total += x; }); + return total / sample.length; +} + +function calculateStandardDeviation(sample, mean) { + var deviation = 0; + sample.forEach(function(x) { + deviation += Math.pow(x - mean, 2); + }); + deviation = deviation / (sample.length); + deviation = Math.sqrt(deviation); + return deviation; +} + +function getRegressionSlope(xValues, xMean, yValues, yMean) { + // See http://en.wikipedia.org/wiki/Simple_linear_regression + var dividendSum = 0; + var divisorSum = 0; + for (var i=0; i