build(docs-infra): support running cli docs examples concurrently (#29261)

PR Close #29261

aio/content/examples/component-interaction/example-config.json

@@ -4,7 +4,8 @@
       "cmd": "yarn",
       "args": [
         "e2e",
-        "--no-webdriver-update"
+        "--no-webdriver-update",
+        "--port={PORT}"
       ]
     }
   ]

aio/content/examples/dependency-injection/example-config.json

@@ -4,7 +4,8 @@
       "cmd": "yarn",
       "args": [
         "e2e",
-        "--no-webdriver-update"
+        "--no-webdriver-update",
+        "--port={PORT}"
       ]
     }
   ]

aio/content/examples/i18n/example-config.json

@@ -5,7 +5,8 @@
       "cmd": "yarn",
       "args": [
         "e2e",
-        "--no-webdriver-update"
+        "--no-webdriver-update",
+        "--port={PORT}"
       ]
     }
   ]

aio/content/examples/service-worker-getting-started/example-config.json

@@ -1,7 +1,7 @@
 {
   "projectType": "service-worker",
   "e2e": [
-    {"cmd": "yarn", "args": ["e2e", "--no-webdriver-update"]},
+    {"cmd": "yarn", "args": ["e2e", "--no-webdriver-update", "--port={PORT}"]},
     {"cmd": "yarn", "args": ["build", "--prod"]},
     {"cmd": "node", "args": ["--eval", "assert(fs.existsSync('./dist/ngsw.json'), 'ngsw.json is missing')"]},
     {"cmd": "node", "args": ["--eval", "assert(fs.existsSync('./dist/ngsw-worker.js'), 'ngsw-worker.js is missing')"]},

aio/package.json

@@ -118,6 +118,7 @@
     "entities": "^1.1.1",
     "eslint": "^3.19.0",
     "eslint-plugin-jasmine": "^2.2.0",
+    "find-free-port": "^2.0.0",
     "firebase-tools": "^5.1.1",
     "fs-extra": "^2.1.2",
     "globby": "^6.1.0",

aio/tools/examples/run-example-e2e.js

@@ -5,6 +5,7 @@ const globby = require('globby');
 const xSpawn = require('cross-spawn');
 const treeKill = require('tree-kill');
 const shelljs = require('shelljs');
+const findFreePort = require('find-free-port');
 
 shelljs.set('-e');
 
@@ -15,6 +16,8 @@ const PROTRACTOR_CONFIG_FILENAME = path.join(__dirname, './shared/protractor.con
 const SJS_SPEC_FILENAME = 'e2e-spec.ts';
 const CLI_SPEC_FILENAME = 'e2e/src/app.e2e-spec.ts';
 const EXAMPLE_CONFIG_FILENAME = 'example-config.json';
+const DEFAULT_CLI_EXAMPLE_PORT = 4200;
+const DEFAULT_CLI_SPECS_CONCURRENCY = 1;
 const IGNORED_EXAMPLES = [
   // temporary ignores
 
@@ -59,6 +62,9 @@ if (argv.ivy) {
  *    e.g. --shard=0/2 // the even specs: 0, 2, 4, etc
  *    e.g. --shard=1/2 // the odd specs: 1, 3, 5, etc
  *    e.g. --shard=1/3 // the second of every three specs: 1, 4, 7, etc
+ *
+ *  --cliSpecsConcurrency Amount of CLI example specs that should be executed concurrently.
+ *    By default runs specs sequentially.
  */
 function runE2e() {
   if (argv.setup) {
@@ -73,7 +79,8 @@ function runE2e() {
   const outputFile = path.join(AIO_PATH, './protractor-results.txt');
 
   return Promise.resolve()
-      .then(() => findAndRunE2eTests(argv.filter, outputFile, argv.shard))
+      .then(() => findAndRunE2eTests(argv.filter, outputFile, argv.shard,
+          argv.cliSpecsConcurrency || DEFAULT_CLI_SPECS_CONCURRENCY))
       .then((status) => {
         reportStatus(status, outputFile);
         if (status.failed.length > 0) {
@@ -88,7 +95,7 @@ function runE2e() {
 
 // Finds all of the *e2e-spec.tests under the examples folder along with the corresponding apps
 // that they should run under. Then run each app/spec collection sequentially.
-function findAndRunE2eTests(filter, outputFile, shard) {
+function findAndRunE2eTests(filter, outputFile, shard, cliSpecsConcurrency) {
   const shardParts = shard ? shard.split('/') : [0, 1];
   const shardModulo = parseInt(shardParts[0], 10);
   const shardDivider = parseInt(shardParts[1], 10);
@@ -99,8 +106,12 @@ function findAndRunE2eTests(filter, outputFile, shard) {
   header += `  Filter: ${filter ? filter : 'All tests'}\n\n`;
   fs.writeFileSync(outputFile, header);
 
-  // Run the tests sequentially.
   const status = {passed: [], failed: []};
+  const updateStatus = (specPath, passed) => {
+    const arr = passed ? status.passed : status.failed;
+    arr.push(specPath);
+  };
+
   return getE2eSpecs(EXAMPLES_PATH, filter)
       .then(e2eSpecPaths => {
         console.log('All e2e specs:');
@@ -119,22 +130,29 @@ function findAndRunE2eTests(filter, outputFile, shard) {
                 (promise, specPath) => {
                   return promise.then(() => {
                     const examplePath = path.dirname(specPath);
-                    return runE2eTestsSystemJS(examplePath, outputFile).then(ok => {
+                    return runE2eTestsSystemJS(examplePath, outputFile)
-                      const arr = ok ? status.passed : status.failed;
+                      .then(passed => updateStatus(examplePath, passed));
-                      arr.push(examplePath);
-                    });
                   });
                 },
                 Promise.resolve())
-            .then(() => {
+            .then(async () => {
-              return e2eSpecPaths.cli.reduce((promise, specPath) => {
+              const specQueue = [...e2eSpecPaths.cli];
-                return promise.then(() => {
+              // Determine free ports for the amount of pending CLI specs before starting
-                  return runE2eTestsCLI(specPath, outputFile).then(ok => {
+              // any tests. This is necessary because ports can stuck in the "TIME_WAIT"
-                    const arr = ok ? status.passed : status.failed;
+              // state after others specs which used that port exited. This works around
-                    arr.push(specPath);
+              // this potential race condition which surfaces on Windows.
-                  });
+              const ports = await findFreePort(4000, 6000, '127.0.0.1', specQueue.length);
-                });
+              // Enable buffering of the process output in case multiple CLI specs will
-              }, Promise.resolve());
+              // be executed concurrently. This means that we can can print out the full
+              // output at once without interfering with other CLI specs printing as well.
+              const bufferOutput = cliSpecsConcurrency > 1;
+              while (specQueue.length) {
+                const chunk = specQueue.splice(0, cliSpecsConcurrency);
+                await Promise.all(chunk.map((testDir, index) => {
+                  return runE2eTestsCLI(testDir, outputFile, bufferOutput, ports.pop())
+                    .then(passed => updateStatus(testDir, passed));
+                }));
+              }
             });
       })
       .then(() => {
@@ -226,29 +244,45 @@ function runProtractorAoT(appDir, outputFile) {
 // fileName; then shut down the example.
 // All protractor output is appended to the outputFile.
 // CLI version
-function runE2eTestsCLI(appDir, outputFile) {
+function runE2eTestsCLI(appDir, outputFile, bufferOutput, port) {
+  if (!bufferOutput) {
     console.log(`\n\n=========== Running aio example tests for: ${appDir}`);
+  }
+
   // `--no-webdriver-update` is needed to preserve the ChromeDriver version already installed.
   const config = loadExampleConfig(appDir);
-  const commands = config.e2e || [{cmd: 'yarn', args: ['e2e', '--prod', '--no-webdriver-update']}];
+  const commands = config.e2e || [{
+    cmd: 'yarn',
+    args: ['e2e', '--prod', '--no-webdriver-update', `--port=${port || DEFAULT_CLI_EXAMPLE_PORT}`]
+  }];
+  let bufferedOutput = `\n\n============== AIO example output for: ${appDir}\n\n`;
 
   const e2eSpawnPromise = commands.reduce((prevSpawnPromise, {cmd, args}) => {
+    // Replace the port placeholder with the specified port if present. Specs that
+    // define their e2e test commands in the example config are able to use the
+    // given available port. This ensures that the CLI tests can be run concurrently.
+    args = args.map(a => a.replace('{PORT}', port || DEFAULT_CLI_EXAMPLE_PORT));
+
     return prevSpawnPromise.then(() => {
-      const currSpawn = spawnExt(cmd, args, {cwd: appDir});
+      const currSpawn = spawnExt(cmd, args, {cwd: appDir}, false,
+          bufferOutput ? msg => bufferedOutput += msg : undefined);
       return currSpawn.promise.then(
           () => Promise.resolve(finish(currSpawn.proc.pid, true)),
           () => Promise.reject(finish(currSpawn.proc.pid, false)));
     });
   }, Promise.resolve());
 
-  return e2eSpawnPromise.then(
+  return e2eSpawnPromise.then(() => {
-      () => {
     fs.appendFileSync(outputFile, `Passed: ${appDir}\n\n`);
     return true;
-      },
+  }, () => {
-      () => {
     fs.appendFileSync(outputFile, `Failed: ${appDir}\n\n`);
     return false;
+  }).then(passed => {
+    if (bufferOutput) {
+      process.stdout.write(bufferedOutput);
+    }
+    return passed;
   });
 }
 
@@ -283,11 +317,13 @@ function reportStatus(status, outputFile) {
 }
 
 // Returns both a promise and the spawned process so that it can be killed if needed.
-function spawnExt(command, args, options, ignoreClose = false) {
+function spawnExt(command, args, options, ignoreClose = false,
+                  printMessage = msg => process.stdout.write(msg)) {
   let proc;
   const promise = new Promise((resolve, reject) => {
     let descr = command + ' ' + args.join(' ');
-    console.log('running: ' + descr);
+    let processOutput = '';
+    printMessage(`running: ${descr}\n`);
     try {
       proc = xSpawn.spawn(command, args, options);
     } catch (e) {
@@ -295,17 +331,18 @@ function spawnExt(command, args, options, ignoreClose = false) {
       reject(e);
       return {proc: null, promise};
     }
-    proc.stdout.on('data', function(data) { process.stdout.write(data.toString()); });
+    proc.stdout.on('data', printMessage);
-    proc.stderr.on('data', function(data) { process.stdout.write(data.toString()); });
+    proc.stderr.on('data', printMessage);
+
     proc.on('close', function(returnCode) {
-      console.log(`completed: ${descr} \n`);
+      printMessage(`completed: ${descr}\n\n`);
       // Many tasks (e.g., tsc) complete but are actually errors;
       // Confirm return code is zero.
       returnCode === 0 || ignoreClose ? resolve(0) : reject(returnCode);
     });
     proc.on('error', function(data) {
-      console.log(`completed with error: ${descr} \n`);
+      printMessage(`completed with error: ${descr}\n\n`);
-      console.log(data.toString());
+      printMessage(`${data.toString()}\n`);
       reject(data);
     });
   });

aio/yarn.lock

@@ -3760,6 +3760,11 @@ find-cache-dir@^2.0.0:
     make-dir "^1.0.0"
     pkg-dir "^3.0.0"
 
+find-free-port@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/find-free-port/-/find-free-port-2.0.0.tgz#4b22e5f6579eb1a38c41ac6bcb3efed1b6da9b1b"
+  integrity sha1-SyLl9leesaOMQaxryz7+0bbamxs=
+
 find-up@^1.0.0:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/find-up/-/find-up-1.1.2.tgz#6b2e9822b1a2ce0a60ab64d610eccad53cb24d0f"