feat(docs-infra): support checking the scores on all audit categories (#31414)

Previously, the `test-pwa-score` script would only check the `pwa`
score. (All categories were reported, but a min. score could only be
specified for `pwa`.)

This commit adds support for checking the scores on all available
categories (such as a11y, performance, seo, etc.).

PR Close #31414

aio/package.json

@@ -33,7 +33,7 @@
     "set-opensearch-url": "node --eval \"const sh = require('shelljs'); sh.set('-e'); sh.sed('-i', /PLACEHOLDER_URL/g, process.argv[1], 'dist/assets/opensearch.xml');\"",
     "presmoke-tests": "yarn update-webdriver",
     "smoke-tests": "protractor tests/deployment/e2e/protractor.conf.js --suite smoke --baseUrl",
-    "test-pwa-score": "node scripts/test-pwa-score",
+    "test-pwa-score": "run-s \"~~audit-web-app {1} all:0,pwa:{2} {3}\" --",
     "test-pwa-score-localhost": "run-p --race \"~~http-server dist -p 4200 --silent\" \"test-pwa-score http://localhost:4200 {1} {2}\" --",
     "example-e2e": "yarn example-check-local && node ./tools/examples/run-example-e2e",
     "example-lint": "tslint --config \"content/examples/tslint.json\" \"content/examples/**/*.ts\" --exclude \"content/examples/styleguide/**/*.avoid.ts\"",
@@ -64,6 +64,7 @@
     "generate-zips": "node ./tools/example-zipper/generateZips",
     "build-404-page": "node scripts/build-404-page",
     "update-webdriver": "webdriver-manager update --standalone false --gecko false $CI_CHROMEDRIVER_VERSION_ARG",
+    "~~audit-web-app": "node scripts/audit-web-app",
     "~~check-env": "node scripts/check-environment",
     "~~clean-generated": "node --eval \"require('shelljs').rm('-rf', 'src/generated')\"",
     "~~build": "ng build --configuration=stable",

aio/scripts/audit-web-app.js

@@ -0,0 +1,179 @@
+#!/bin/env node
+'use strict';
+
+/**
+ * Usage:
+ * ```sh
+ * node scripts/audit-web-app <url> <min-scores> [<log-file>]
+ * ```
+ *
+ * Runs audits against the specified URL on specific categories (accessibility, best practices, performance, PWA, SEO).
+ * It fails, if the score in any category is below the score specified in `<min-scores>`. (Only runs audits for the
+ * specified categories.)
+ *
+ * `<min-scores>` is either a number (in which case it is interpreted as `all:<min-score>`) or a list of comma-separated
+ * strings of the form `key:value`, where `key` is one of `accessibility`, `best-practices`, `performance`, `pwa`, `seo`
+ * or `all` and `value` is a number (between 0 and 100).
+ *
+ * Examples:
+ * - `95` _(Same as `all:95`.)_
+ * - `all:95` _(Run audits for all categories and require a score of 95 or higher.)_
+ * - `all:95,pwa:100` _(Same as `all:95`, except that a scope of 100 is required for the `pwa` category.)_
+ * - `performance:90` _(Only run audits for the `performance` category and require a score of 90 or higher.)_
+ *
+ * If `<log-file>` is defined, the full results will be logged there.
+ *
+ * (Skips HTTPS-related audits, when run for an HTTP URL.)
+ */
+
+// Imports
+const chromeLauncher = require('chrome-launcher');
+const lighthouse = require('lighthouse');
+const printer = require('lighthouse/lighthouse-cli/printer');
+const logger = require('lighthouse-logger');
+
+// Constants
+const AUDIT_CATEGORIES = ['accessibility', 'best-practices', 'performance', 'pwa', 'seo'];
+const CHROME_LAUNCH_OPTS = {chromeFlags: ['--headless']};
+const LIGHTHOUSE_FLAGS = {logLevel: process.env.CI ? 'error' : 'info'};  // Be less verbose on CI.
+const SKIPPED_HTTPS_AUDITS = ['redirects-http', 'uses-http2'];
+const VIEWER_URL = 'https://googlechrome.github.io/lighthouse/viewer';
+const WAIT_FOR_SW_DELAY = 5000;
+
+// Run
+_main(process.argv.slice(2));
+
+// Functions - Definitions
+async function _main(args) {
+  const {url, minScores, logFile} = parseInput(args);
+  const isOnHttp = /^http:/.test(url);
+  const lhFlags = {...LIGHTHOUSE_FLAGS, onlyCategories: Object.keys(minScores).sort()};
+  const lhConfig = {
+    extends: 'lighthouse:default',
+    // Since the Angular ServiceWorker waits for the app to stabilize before registering,
+    // wait a few seconds after load to allow Lighthouse to reliably detect it.
+    passes: [{passName: 'defaultPass', pauseAfterLoadMs: WAIT_FOR_SW_DELAY}],
+  };
+
+  console.log(`Running web-app audits for '${url}'...`);
+  console.log(`  Audit categories: ${lhFlags.onlyCategories.join(', ')}`);
+
+  // If testing on HTTP, skip HTTPS-specific tests.
+  // (Note: Browsers special-case localhost and run ServiceWorker even on HTTP.)
+  if (isOnHttp) skipHttpsAudits(lhConfig);
+
+  logger.setLevel(lhFlags.logLevel);
+
+  try {
+    console.log('');
+    const startTime = Date.now();
+    const results = await launchChromeAndRunLighthouse(url, lhFlags, lhConfig);
+    const success = await processResults(results, minScores, logFile);
+    console.log(`\n(Completed in ${((Date.now() - startTime) / 1000).toFixed(1)}s.)\n`);
+
+    if (!success) {
+      throw new Error('One or more scores are too low.');
+    }
+  } catch (err) {
+    onError(err);
+  }
+}
+
+function formatScore(score) {
+  return `${(score * 100).toFixed(0).padStart(3)}`;
+}
+
+async function launchChromeAndRunLighthouse(url, flags, config) {
+  const chrome = await chromeLauncher.launch(CHROME_LAUNCH_OPTS);
+  flags.port = chrome.port;
+
+  try {
+    return await lighthouse(url, flags, config);
+  } finally {
+    await chrome.kill();
+  }
+}
+
+function onError(err) {
+  console.error(err);
+  console.error('');
+  process.exit(1);
+}
+
+function parseInput(args) {
+  const [url, minScoresRaw, logFile] = args;
+
+  if (!url) {
+    onError('Invalid arguments: <url> not specified.');
+  } else if (!minScoresRaw) {
+    onError('Invalid arguments: <min-scores> not specified.');
+  }
+
+  const minScores = parseMinScores(minScoresRaw || '');
+  const unknownCategories = Object.keys(minScores).filter(cat => !AUDIT_CATEGORIES.includes(cat));
+  const allValuesValid = Object.values(minScores).every(x => (0 <= x) && (x <= 1));
+
+  if (unknownCategories.length > 0) {
+    onError(`Invalid arguments: <min-scores> contains unknown category(-ies): ${unknownCategories.join(', ')}`);
+  } else if (!allValuesValid) {
+    onError(`Invalid arguments: <min-scores> has non-numeric or out-of-range values: ${minScoresRaw}`);
+  }
+
+  return {url, minScores, logFile};
+}
+
+function parseMinScores(raw) {
+  const minScores = {};
+
+  if (/^\d+$/.test(raw)) {
+    raw = `all:${raw}`;
+  }
+
+  raw.
+    split(',').
+    map(x => x.split(':')).
+    forEach(([key, val]) => minScores[key] = Number(val) / 100);
+
+  if (minScores.hasOwnProperty('all')) {
+    AUDIT_CATEGORIES.forEach(cat => minScores.hasOwnProperty(cat) || (minScores[cat] = minScores.all));
+    delete minScores.all;
+  }
+
+  return minScores;
+}
+
+async function processResults(results, minScores, logFile) {
+  const lhVersion = results.lhr.lighthouseVersion;
+  const categories = results.lhr.categories;
+  const report = results.report;
+
+  if (logFile) {
+    console.log(`\nSaving results in '${logFile}'...`);
+    console.log(`  LightHouse viewer: ${VIEWER_URL}`);
+
+    await printer.write(report, printer.OutputMode.json, logFile);
+  }
+
+  console.log(`\nLighthouse version: ${lhVersion}`);
+  console.log('\nAudit results:');
+
+  const maxTitleLen = Math.max(...Object.values(categories).map(({title}) => title.length));
+  const success = Object.keys(categories).sort().reduce((aggr, cat) => {
+    const {title, score} = categories[cat];
+    const paddedTitle = `${title}:`.padEnd(maxTitleLen + 1);
+    const minScore = minScores[cat];
+    const passed = !isNaN(score) && (score >= minScore);
+
+    console.log(
+      `  - ${paddedTitle}  ${formatScore(score)}  (Required: ${formatScore(minScore)})  ${passed ? 'OK' : 'FAILED'}`);
+
+    return aggr && passed;
+  }, true);
+
+  return success;
+}
+
+function skipHttpsAudits(config) {
+  console.log(`  Skipping HTTPS-related audits: ${SKIPPED_HTTPS_AUDITS.join(', ')}`);
+  config.settings = {...config.settings, skipAudits: SKIPPED_HTTPS_AUDITS};
+}

aio/scripts/test-pwa-score.js

@@ -1,142 +0,0 @@
-#!/bin/env node
-'use strict';
-
-/**
- * Usage:
- * ```sh
- * node scripts/test-pwa-score <url> <min-score> [<log-file>]
- * ```
- *
- * Fails if the score is below `<min-score>`.
- * If `<log-file>` is defined, the full results will be logged there.
- *
- * (Skips HTTPS-related audits, when run for an HTTP URL.)
- */
-
-// Imports
-const chromeLauncher = require('chrome-launcher');
-const lighthouse = require('lighthouse');
-const printer = require('lighthouse/lighthouse-cli/printer');
-const logger = require('lighthouse-logger');
-
-// Constants
-const CHROME_LAUNCH_OPTS = {chromeFlags: ['--headless']};
-const LIGHTHOUSE_FLAGS = {logLevel: process.env.CI ? 'error' : 'info'};  // Be less verbose on CI.
-const SKIPPED_HTTPS_AUDITS = ['redirects-http', 'uses-http2'];
-const VIEWER_URL = 'https://googlechrome.github.io/lighthouse/viewer';
-const WAIT_FOR_SW_DELAY = 5000;
-
-// Run
-_main(process.argv.slice(2));
-
-// Functions - Definitions
-async function _main(args) {
-  const {url, minScore, logFile} = parseInput(args);
-  const isOnHttp = /^http:/.test(url);
-  const config = {
-    extends: 'lighthouse:default',
-    // Since the Angular ServiceWorker waits for the app to stabilize before registering,
-    // wait a few seconds after load to allow Lighthouse to reliably detect it.
-    passes: [{passName: 'defaultPass', pauseAfterLoadMs: WAIT_FOR_SW_DELAY}],
-  };
-
-  console.log(`Running PWA audit for '${url}'...`);
-
-  // If testing on HTTP, skip HTTPS-specific tests.
-  // (Note: Browsers special-case localhost and run ServiceWorker even on HTTP.)
-  if (isOnHttp) skipHttpsAudits(config);
-
-  logger.setLevel(LIGHTHOUSE_FLAGS.logLevel);
-
-  try {
-    console.log('');
-    const startTime = Date.now();
-    const results = await launchChromeAndRunLighthouse(url, LIGHTHOUSE_FLAGS, config);
-    const score = await processResults(results, logFile);
-    evaluateScore(minScore, score);
-    console.log(`\n(Completed in ${((Date.now() - startTime) / 1000).toFixed(1)}s.)\n`);
-  } catch (err) {
-    onError(err);
-  }
-}
-
-function evaluateScore(expectedScore, actualScore) {
-  console.log('\nLighthouse PWA score:');
-  console.log(`  - Expected: ${formatScore(expectedScore)} (or higher)`);
-  console.log(`  - Actual:   ${formatScore(actualScore)}`);
-
-  if (isNaN(actualScore) || (actualScore < expectedScore)) {
-    throw new Error(`PWA score is too low. (${actualScore} < ${expectedScore})`);
-  }
-}
-
-function formatScore(score) {
-  return `${(score * 100).toFixed(0).padStart(3)}`;
-}
-
-async function launchChromeAndRunLighthouse(url, flags, config) {
-  const chrome = await chromeLauncher.launch(CHROME_LAUNCH_OPTS);
-  flags.port = chrome.port;
-
-  try {
-    return await lighthouse(url, flags, config);
-  } finally {
-    await chrome.kill();
-  }
-}
-
-function onError(err) {
-  console.error(err);
-  console.error('');
-  process.exit(1);
-}
-
-function parseInput(args) {
-  const [url, minScoreRaw, logFile] = args;
-
-  if (!url) {
-    onError('Invalid arguments: <url> not specified.');
-  } else if (!minScoreRaw) {
-    onError('Invalid arguments: <min-score> not specified.');
-  }
-
-  const minScore = Number(minScoreRaw) / 100;
-  const isValid = (0 <= minScore) && (minScore <= 1);
-
-  if (!isValid) {
-    onError(`Invalid arguments: <min-score> has non-numeric or out-of-range values: ${minScoreRaw}`);
-  }
-
-  return {url, minScore, logFile};
-}
-
-async function processResults(results, logFile) {
-  const lhVersion = results.lhr.lighthouseVersion;
-  const categories = results.lhr.categories;
-  const report = results.report;
-
-  if (logFile) {
-    console.log(`\nSaving results in '${logFile}'...`);
-    console.log(`  LightHouse viewer: ${VIEWER_URL}`);
-
-    await printer.write(report, printer.OutputMode.json, logFile);
-  }
-
-  console.log(`\nLighthouse version: ${lhVersion}`);
-  console.log('\nAudit scores:');
-
-  const maxTitleLen = Math.max(...Object.values(categories).map(({title}) => title.length));
-  Object.keys(categories).sort().forEach(cat => {
-    const {title, score} = categories[cat];
-    const paddedTitle = `${title}:`.padEnd(maxTitleLen + 1);
-
-    console.log(`  - ${paddedTitle}  ${formatScore(score)}`);
-  });
-
-  return categories.pwa.score;
-}
-
-function skipHttpsAudits(config) {
-  console.log(`  Skipping HTTPS-related audits: ${SKIPPED_HTTPS_AUDITS.join(', ')}`);
-  config.settings = {...config.settings, skipAudits: SKIPPED_HTTPS_AUDITS};
-}