diff --git a/protractor-perf-shared.js b/protractor-perf-shared.js index 26982f326c..6cb2568e09 100644 --- a/protractor-perf-shared.js +++ b/protractor-perf-shared.js @@ -1,6 +1,7 @@ var config = exports.config = require('./protractor-shared.js').config; // load traceur runtime as our tests are written in es6 require('traceur/bin/traceur-runtime.js'); +var nodeUuid = require('node-uuid'); var cloudReporterConfig; if (process.env.CLOUD_SECRET_PATH) { @@ -8,15 +9,22 @@ if (process.env.CLOUD_SECRET_PATH) { cloudReporterConfig = { auth: require(process.env.CLOUD_SECRET_PATH), projectId: 'angular-perf', - datasetId: 'benchmarks' + datasetId: 'benchmarks', + tableId: 'ng2perf' }; } config.specs = ['dist/cjs/**/*_perf.js']; config.jasmineNodeOpts.defaultTimeoutInterval = 80000; +var runId = nodeUuid.v1(); +if (process.env.GIT_SHA) { + runId = process.env.GIT_SHA + ' ' + runId; +} + config.params = { benchmark: { + runId: runId, // size of the sample to take sampleSize: 20, timeout: 60000, diff --git a/scripts/jenkins/jenkins_perf.sh b/scripts/jenkins/jenkins_perf.sh index 9cde3bd043..e897945298 100755 --- a/scripts/jenkins/jenkins_perf.sh +++ b/scripts/jenkins/jenkins_perf.sh @@ -10,6 +10,7 @@ export CHANNEL=stable export ARCH=macos-ia32 export PERF_BROWSERS=ChromeAndroid export CLOUD_SECRET_PATH="/Users/Shared/jenkins/keys/perf-cloud-secret" +export GIT_SHA=$(git rev-parse HEAD) nvm use 0.10 diff --git a/tools/benchpress/src/benchmark.es6 b/tools/benchpress/src/benchmark.es6 index 526eb9b995..8320bc8ee3 100644 --- a/tools/benchpress/src/benchmark.es6 +++ b/tools/benchpress/src/benchmark.es6 @@ -1,6 +1,5 @@ var statistics = require('./statistics'); var commands = require('./commands'); -var nodeUuid = require('node-uuid'); var webdriver = require('protractor/node_modules/selenium-webdriver'); var SUPPORTED_METRICS = { @@ -21,11 +20,10 @@ module.exports = { }; function runBenchmark(config, workCallback) { - var sampleId = nodeUuid.v1(); var reporters = config.reporters.filter(function(Class) { return !!Class; }).map(function(Class) { - return new Class(sampleId, config); + return new Class(config); }); var scriptMetricIndex = -1; config.metrics.forEach(function(metric, index) { diff --git a/tools/benchpress/src/cloud_reporter.es6 b/tools/benchpress/src/cloud_reporter.es6 index 9b2263c71f..58fe0f9993 100644 --- a/tools/benchpress/src/cloud_reporter.es6 +++ b/tools/benchpress/src/cloud_reporter.es6 @@ -2,11 +2,16 @@ var google = require('googleapis'); var bigquery = google.bigquery('v2'); var webdriver = require('protractor/node_modules/selenium-webdriver'); -var HEADER_FIELDS = [ +var TABLE_FIELDS = [ { "name": 'runId', "type": 'STRING', - "description": 'uuid for the benchmark run' + "description": 'git SHA and uuid for the benchmark run' + }, + { + "name": 'benchmarkId', + "type": 'STRING', + "description": 'id of the benchmark' }, { "name": 'index', @@ -26,22 +31,65 @@ var HEADER_FIELDS = [ "name": 'forceGc', "type": 'BOOLEAN', "description": 'whether gc was forced at end of action' + }, + { + "name": 'stable', + "type": 'BOOLEAN', + "description": 'whether this entry was part of the stable sample' + }, + { + "name": 'params', + "type": 'RECORD', + "description": 'parameters of the benchmark', + "mode": 'REPEATED', + "fields": [ + { + "name": 'name', + "type": 'STRING', + "description": 'param name' + }, + { + "name": 'strvalue', + "type": 'STRING', + "description": 'param value for strings' + }, + { + "name": 'numvalue', + "type": 'FLOAT', + "description": 'param value for numbers' + } + ] + }, + { + "name": 'metrics', + "type": 'RECORD', + "description": 'metrics of the benchmark', + "mode": 'REPEATED', + "fields": [ + { + "name": 'name', + "type": 'STRING', + "description": 'metric name' + }, + { + "name": 'value', + "type": 'FLOAT', + "description": 'metric value' + } + ] } ]; class CloudReporter { - constructor(runId, benchmarkConfig) { - this.stableRowsTableConfig = createTableConfig(benchmarkConfig, '_stable'); - this.allRowsTableConfig = createTableConfig(benchmarkConfig, '_all') + constructor(benchmarkConfig) { + this.tableConfig = createTableConfig(benchmarkConfig); this.authConfig = benchmarkConfig.cloudReporter.auth; this.benchmarkConfig = benchmarkConfig; - this.runId = runId; - this.allRows = []; + this.allSample = []; var self = this; browser.executeScript('return navigator.userAgent').then(function(userAgent) { self.browserUserAgent = userAgent; }); - } begin() { var self = this; @@ -52,66 +100,62 @@ class CloudReporter { }); }); flow.execute(function() { - return webdriver.promise.all([ - getOrCreateTable(self.authClient, self.allRowsTableConfig), - getOrCreateTable(self.authClient, self.stableRowsTableConfig) - ]); + return getOrCreateTable(self.authClient, self.tableConfig); }); } add(data) { - this.allRows.push(this._convertToTableRow(data)); + this.allSample.push(data); } end(stableSample) { var self = this; var flow = browser.driver.controlFlow(); - var stableRows = stableSample.map(function(data) { - return self._convertToTableRow(data); + var allRows = this.allSample.map(function(data) { + return self._convertToTableRow(data, stableSample); }); flow.execute(function() { - return webdriver.promise.all([ - insertRows(self.authClient, self.stableRowsTableConfig, stableRows), - insertRows(self.authClient, self.allRowsTableConfig, self.allRows) - ]); + return insertRows(self.authClient, self.tableConfig, allRows) }); } - _convertToTableRow(benchpressRow) { + _convertToTableRow(benchpressRow, stableSample) { var tableRow = { - runId: this.runId, + runId: this.benchmarkConfig.runId, + benchmarkId: this.benchmarkConfig.id, index: benchpressRow.index, creationTime: new Date(), browser: this.browserUserAgent, - forceGc: benchpressRow.forceGc + forceGc: benchpressRow.forceGc, + stable: stableSample.indexOf(benchpressRow) >= 0, + params: this.benchmarkConfig.params.map(function(param) { + if (typeof param.value === 'number') { + return { + name: param.name, + numvalue: param.value + }; + } else { + return { + name: param.name, + strvalue: ''+param.value + } + } + }), + metrics: this.benchmarkConfig.metrics.map(function(metricName, index) { + return { + name: metricName, + value: benchpressRow.values[index] + }; + }) }; - this.benchmarkConfig.params.forEach(function(param) { - tableRow['p_'+param.name] = param.value; - }); - this.benchmarkConfig.metrics.forEach(function(metric, index) { - tableRow['m_'+metric] = benchpressRow.values[index]; - }); return tableRow; } } -function createTableConfig(benchmarkConfig, tableSuffix) { - var tableId = (benchmarkConfig.id+tableSuffix).replace(/\./g, '_'); +function createTableConfig(benchmarkConfig) { return { projectId: benchmarkConfig.cloudReporter.projectId, datasetId: benchmarkConfig.cloudReporter.datasetId, table: { - id: tableId, - fields: HEADER_FIELDS - .concat(benchmarkConfig.params.map(function(param) { - return { - "name": 'p_'+param.name, - "type": 'FLOAT' - }; - })) - .concat(benchmarkConfig.metrics.map(function(metricName) { - return { - "name": 'm_'+metricName, - "type": 'FLOAT' - }; - })) + id: benchmarkConfig.cloudReporter.tableId, + fields: TABLE_FIELDS } }; } diff --git a/tools/benchpress/src/console_reporter.es6 b/tools/benchpress/src/console_reporter.es6 index e878b27615..8b4cbfe49f 100644 --- a/tools/benchpress/src/console_reporter.es6 +++ b/tools/benchpress/src/console_reporter.es6 @@ -5,9 +5,8 @@ var HEADER_SEPARATORS = ['----', '----', '----', '----', '----', '----', '----'] var FOOTER_SEPARATORS = ['====', '====', '====', '====', '====', '====', '====']; class ConsoleReporter { - constructor(runId, config) { + constructor(config) { this.config = config; - this.runId = runId; this.rowFormat = ['%12s'].concat(config.metrics.map(function() { return '%12s'; })).join(' | '); @@ -15,7 +14,7 @@ class ConsoleReporter { begin() { printHeading('BENCHMARK '+this.config.id); console.log('sample size', this.config.sampleSize); - console.log('run id', this.runId); + console.log('run id', this.config.runId); console.log('params', JSON.stringify(this.config.params, null, ' ')); printTableHeader(this.rowFormat, ['index', 'forceGc'].concat(this.config.metrics)); }