diff --git a/aio/content/examples/component-interaction/example-config.json b/aio/content/examples/component-interaction/example-config.json index b93fc9e8da..054f3cca48 100644 --- a/aio/content/examples/component-interaction/example-config.json +++ b/aio/content/examples/component-interaction/example-config.json @@ -4,7 +4,8 @@ "cmd": "yarn", "args": [ "e2e", - "--no-webdriver-update" + "--no-webdriver-update", + "--port={PORT}" ] } ] diff --git a/aio/content/examples/dependency-injection/example-config.json b/aio/content/examples/dependency-injection/example-config.json index b93fc9e8da..054f3cca48 100644 --- a/aio/content/examples/dependency-injection/example-config.json +++ b/aio/content/examples/dependency-injection/example-config.json @@ -4,7 +4,8 @@ "cmd": "yarn", "args": [ "e2e", - "--no-webdriver-update" + "--no-webdriver-update", + "--port={PORT}" ] } ] diff --git a/aio/content/examples/i18n/example-config.json b/aio/content/examples/i18n/example-config.json index 99b45f2c49..4b2197f432 100644 --- a/aio/content/examples/i18n/example-config.json +++ b/aio/content/examples/i18n/example-config.json @@ -5,7 +5,8 @@ "cmd": "yarn", "args": [ "e2e", - "--no-webdriver-update" + "--no-webdriver-update", + "--port={PORT}" ] } ] diff --git a/aio/content/examples/service-worker-getting-started/example-config.json b/aio/content/examples/service-worker-getting-started/example-config.json index b0e7e5672e..0ebe5bd24e 100644 --- a/aio/content/examples/service-worker-getting-started/example-config.json +++ b/aio/content/examples/service-worker-getting-started/example-config.json @@ -1,7 +1,7 @@ { "projectType": "service-worker", "e2e": [ - {"cmd": "yarn", "args": ["e2e", "--no-webdriver-update"]}, + {"cmd": "yarn", "args": ["e2e", "--no-webdriver-update", "--port={PORT}"]}, {"cmd": "yarn", "args": ["build", "--prod"]}, {"cmd": "node", "args": ["--eval", "assert(fs.existsSync('./dist/ngsw.json'), 'ngsw.json is missing')"]}, {"cmd": "node", "args": ["--eval", "assert(fs.existsSync('./dist/ngsw-worker.js'), 'ngsw-worker.js is missing')"]}, diff --git a/aio/package.json b/aio/package.json index bfbb1566d7..82ad678a03 100644 --- a/aio/package.json +++ b/aio/package.json @@ -118,6 +118,7 @@ "entities": "^1.1.1", "eslint": "^3.19.0", "eslint-plugin-jasmine": "^2.2.0", + "find-free-port": "^2.0.0", "firebase-tools": "^5.1.1", "fs-extra": "^2.1.2", "globby": "^6.1.0", diff --git a/aio/tools/examples/run-example-e2e.js b/aio/tools/examples/run-example-e2e.js index d3d08393f4..12516c0958 100644 --- a/aio/tools/examples/run-example-e2e.js +++ b/aio/tools/examples/run-example-e2e.js @@ -5,6 +5,7 @@ const globby = require('globby'); const xSpawn = require('cross-spawn'); const treeKill = require('tree-kill'); const shelljs = require('shelljs'); +const findFreePort = require('find-free-port'); shelljs.set('-e'); @@ -15,6 +16,8 @@ const PROTRACTOR_CONFIG_FILENAME = path.join(__dirname, './shared/protractor.con const SJS_SPEC_FILENAME = 'e2e-spec.ts'; const CLI_SPEC_FILENAME = 'e2e/src/app.e2e-spec.ts'; const EXAMPLE_CONFIG_FILENAME = 'example-config.json'; +const DEFAULT_CLI_EXAMPLE_PORT = 4200; +const DEFAULT_CLI_SPECS_CONCURRENCY = 1; const IGNORED_EXAMPLES = [ // temporary ignores @@ -59,6 +62,9 @@ if (argv.ivy) { * e.g. --shard=0/2 // the even specs: 0, 2, 4, etc * e.g. --shard=1/2 // the odd specs: 1, 3, 5, etc * e.g. --shard=1/3 // the second of every three specs: 1, 4, 7, etc + * + * --cliSpecsConcurrency Amount of CLI example specs that should be executed concurrently. + * By default runs specs sequentially. */ function runE2e() { if (argv.setup) { @@ -73,7 +79,8 @@ function runE2e() { const outputFile = path.join(AIO_PATH, './protractor-results.txt'); return Promise.resolve() - .then(() => findAndRunE2eTests(argv.filter, outputFile, argv.shard)) + .then(() => findAndRunE2eTests(argv.filter, outputFile, argv.shard, + argv.cliSpecsConcurrency || DEFAULT_CLI_SPECS_CONCURRENCY)) .then((status) => { reportStatus(status, outputFile); if (status.failed.length > 0) { @@ -88,7 +95,7 @@ function runE2e() { // Finds all of the *e2e-spec.tests under the examples folder along with the corresponding apps // that they should run under. Then run each app/spec collection sequentially. -function findAndRunE2eTests(filter, outputFile, shard) { +function findAndRunE2eTests(filter, outputFile, shard, cliSpecsConcurrency) { const shardParts = shard ? shard.split('/') : [0, 1]; const shardModulo = parseInt(shardParts[0], 10); const shardDivider = parseInt(shardParts[1], 10); @@ -99,8 +106,12 @@ function findAndRunE2eTests(filter, outputFile, shard) { header += ` Filter: ${filter ? filter : 'All tests'}\n\n`; fs.writeFileSync(outputFile, header); - // Run the tests sequentially. const status = {passed: [], failed: []}; + const updateStatus = (specPath, passed) => { + const arr = passed ? status.passed : status.failed; + arr.push(specPath); + }; + return getE2eSpecs(EXAMPLES_PATH, filter) .then(e2eSpecPaths => { console.log('All e2e specs:'); @@ -119,22 +130,29 @@ function findAndRunE2eTests(filter, outputFile, shard) { (promise, specPath) => { return promise.then(() => { const examplePath = path.dirname(specPath); - return runE2eTestsSystemJS(examplePath, outputFile).then(ok => { - const arr = ok ? status.passed : status.failed; - arr.push(examplePath); - }); + return runE2eTestsSystemJS(examplePath, outputFile) + .then(passed => updateStatus(examplePath, passed)); }); }, Promise.resolve()) - .then(() => { - return e2eSpecPaths.cli.reduce((promise, specPath) => { - return promise.then(() => { - return runE2eTestsCLI(specPath, outputFile).then(ok => { - const arr = ok ? status.passed : status.failed; - arr.push(specPath); - }); - }); - }, Promise.resolve()); + .then(async () => { + const specQueue = [...e2eSpecPaths.cli]; + // Determine free ports for the amount of pending CLI specs before starting + // any tests. This is necessary because ports can stuck in the "TIME_WAIT" + // state after others specs which used that port exited. This works around + // this potential race condition which surfaces on Windows. + const ports = await findFreePort(4000, 6000, '127.0.0.1', specQueue.length); + // Enable buffering of the process output in case multiple CLI specs will + // be executed concurrently. This means that we can can print out the full + // output at once without interfering with other CLI specs printing as well. + const bufferOutput = cliSpecsConcurrency > 1; + while (specQueue.length) { + const chunk = specQueue.splice(0, cliSpecsConcurrency); + await Promise.all(chunk.map((testDir, index) => { + return runE2eTestsCLI(testDir, outputFile, bufferOutput, ports.pop()) + .then(passed => updateStatus(testDir, passed)); + })); + } }); }) .then(() => { @@ -226,30 +244,46 @@ function runProtractorAoT(appDir, outputFile) { // fileName; then shut down the example. // All protractor output is appended to the outputFile. // CLI version -function runE2eTestsCLI(appDir, outputFile) { - console.log(`\n\n=========== Running aio example tests for: ${appDir}`); +function runE2eTestsCLI(appDir, outputFile, bufferOutput, port) { + if (!bufferOutput) { + console.log(`\n\n=========== Running aio example tests for: ${appDir}`); + } + // `--no-webdriver-update` is needed to preserve the ChromeDriver version already installed. const config = loadExampleConfig(appDir); - const commands = config.e2e || [{cmd: 'yarn', args: ['e2e', '--prod', '--no-webdriver-update']}]; + const commands = config.e2e || [{ + cmd: 'yarn', + args: ['e2e', '--prod', '--no-webdriver-update', `--port=${port || DEFAULT_CLI_EXAMPLE_PORT}`] + }]; + let bufferedOutput = `\n\n============== AIO example output for: ${appDir}\n\n`; const e2eSpawnPromise = commands.reduce((prevSpawnPromise, {cmd, args}) => { + // Replace the port placeholder with the specified port if present. Specs that + // define their e2e test commands in the example config are able to use the + // given available port. This ensures that the CLI tests can be run concurrently. + args = args.map(a => a.replace('{PORT}', port || DEFAULT_CLI_EXAMPLE_PORT)); + return prevSpawnPromise.then(() => { - const currSpawn = spawnExt(cmd, args, {cwd: appDir}); + const currSpawn = spawnExt(cmd, args, {cwd: appDir}, false, + bufferOutput ? msg => bufferedOutput += msg : undefined); return currSpawn.promise.then( () => Promise.resolve(finish(currSpawn.proc.pid, true)), () => Promise.reject(finish(currSpawn.proc.pid, false))); }); }, Promise.resolve()); - return e2eSpawnPromise.then( - () => { - fs.appendFileSync(outputFile, `Passed: ${appDir}\n\n`); - return true; - }, - () => { - fs.appendFileSync(outputFile, `Failed: ${appDir}\n\n`); - return false; - }); + return e2eSpawnPromise.then(() => { + fs.appendFileSync(outputFile, `Passed: ${appDir}\n\n`); + return true; + }, () => { + fs.appendFileSync(outputFile, `Failed: ${appDir}\n\n`); + return false; + }).then(passed => { + if (bufferOutput) { + process.stdout.write(bufferedOutput); + } + return passed; + }); } // Report final status. @@ -283,11 +317,13 @@ function reportStatus(status, outputFile) { } // Returns both a promise and the spawned process so that it can be killed if needed. -function spawnExt(command, args, options, ignoreClose = false) { +function spawnExt(command, args, options, ignoreClose = false, + printMessage = msg => process.stdout.write(msg)) { let proc; const promise = new Promise((resolve, reject) => { let descr = command + ' ' + args.join(' '); - console.log('running: ' + descr); + let processOutput = ''; + printMessage(`running: ${descr}\n`); try { proc = xSpawn.spawn(command, args, options); } catch (e) { @@ -295,17 +331,18 @@ function spawnExt(command, args, options, ignoreClose = false) { reject(e); return {proc: null, promise}; } - proc.stdout.on('data', function(data) { process.stdout.write(data.toString()); }); - proc.stderr.on('data', function(data) { process.stdout.write(data.toString()); }); + proc.stdout.on('data', printMessage); + proc.stderr.on('data', printMessage); + proc.on('close', function(returnCode) { - console.log(`completed: ${descr} \n`); + printMessage(`completed: ${descr}\n\n`); // Many tasks (e.g., tsc) complete but are actually errors; // Confirm return code is zero. returnCode === 0 || ignoreClose ? resolve(0) : reject(returnCode); }); proc.on('error', function(data) { - console.log(`completed with error: ${descr} \n`); - console.log(data.toString()); + printMessage(`completed with error: ${descr}\n\n`); + printMessage(`${data.toString()}\n`); reject(data); }); }); diff --git a/aio/yarn.lock b/aio/yarn.lock index 81fdc4317f..00657598dc 100644 --- a/aio/yarn.lock +++ b/aio/yarn.lock @@ -3760,6 +3760,11 @@ find-cache-dir@^2.0.0: make-dir "^1.0.0" pkg-dir "^3.0.0" +find-free-port@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/find-free-port/-/find-free-port-2.0.0.tgz#4b22e5f6579eb1a38c41ac6bcb3efed1b6da9b1b" + integrity sha1-SyLl9leesaOMQaxryz7+0bbamxs= + find-up@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/find-up/-/find-up-1.1.2.tgz#6b2e9822b1a2ce0a60ab64d610eccad53cb24d0f"