feat(service-worker): introduce the @angular/service-worker package (#19274)

This service worker is a conceptual derivative of the existing @angular/service-worker maintained at github.com/angular/mobile-toolkit, but has been rewritten to support use across a much wider variety of applications.

Entrypoints include:

@angular/service-worker: a library for use within Angular client apps to communicate with the service worker.
@angular/service-worker/gen: a library for generating ngsw.json files from glob-based SW config files.
@angular/service-worker/ngsw-worker.js: the bundled service worker script itself.
@angular/service-worker/ngsw-cli.js: a CLI tool for generating ngsw.json files from glob-based SW config files.

packages/service-worker/config/index.ts

@@ -0,0 +1,14 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+// This file is not used to build this module. It is only used during editing
+// by the TypeScript language service and during build for verification. `ngc`
+// replaces this file with production index.ts when it rewrites private symbol
+// names.
+
+export * from './public_api';

packages/service-worker/config/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@angular/service-worker/config",
+  "typings": "./index.d.ts",
+  "main": "../bundles/service-worker-config.umd.js",
+  "module": "../esm5/config/index.js",
+  "es2015": "../esm15/config/index.js"
+}

packages/service-worker/config/public_api.ts

@@ -0,0 +1,11 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+export {Filesystem} from './src/filesystem';
+export {Generator} from './src/generator';
+export {AssetGroup, Config, DataGroup, Duration, Glob} from './src/in';

packages/service-worker/config/rollup.config.js

@@ -0,0 +1,23 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import resolve from 'rollup-plugin-node-resolve';
+import sourcemaps from 'rollup-plugin-sourcemaps';
+
+const globals = {};
+
+export default {
+  entry: '../../../dist/packages-dist/service-worker/esm5/config.js',
+  dest: '../../../dist/packages-dist/service-worker/bundles/service-worker-config.umd.js',
+  format: 'umd',
+  exports: 'named',
+  moduleName: 'ng.serviceWorker.config',
+  plugins: [resolve(), sourcemaps()],
+  external: Object.keys(globals),
+  globals: globals
+};

packages/service-worker/config/src/duration.ts

@@ -0,0 +1,48 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+const PARSE_TO_PAIRS = /([0-9]+[^0-9]+)/g;
+const PAIR_SPLIT = /^([0-9]+)([dhmsu]+)$/;
+
+export function parseDurationToMs(duration: string): number {
+  const matches: string[] = [];
+
+  let array: RegExpExecArray|null;
+  while ((array = PARSE_TO_PAIRS.exec(duration)) !== null) {
+    matches.push(array[0]);
+  }
+  return matches
+      .map(match => {
+        const res = PAIR_SPLIT.exec(match);
+        if (res === null) {
+          throw new Error(`Not a valid duration: ${match}`);
+        }
+        let factor: number = 0;
+        switch (res[2]) {
+          case 'd':
+            factor = 86400000;
+            break;
+          case 'h':
+            factor = 3600000;
+            break;
+          case 'm':
+            factor = 60000;
+            break;
+          case 's':
+            factor = 1000;
+            break;
+          case 'u':
+            factor = 1;
+            break;
+          default:
+            throw new Error(`Not a valid duration unit: ${res[2]}`);
+        }
+        return parseInt(res[1]) * factor;
+      })
+      .reduce((total, value) => total + value, 0);
+}

packages/service-worker/config/src/filesystem.ts

@@ -0,0 +1,19 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+/**
+ * An abstraction over a virtual file system used to enable testing and operation
+ * of the config generator in different environments.
+ *
+ * @experimental
+ */
+export interface Filesystem {
+  list(dir: string): Promise<string[]>;
+  read(file: string): Promise<string>;
+  write(file: string, contents: string): Promise<void>;
+}

packages/service-worker/config/src/generator.ts

@@ -0,0 +1,134 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {parseDurationToMs} from './duration';
+import {Filesystem} from './filesystem';
+import {globToRegex} from './glob';
+import {Config} from './in';
+import {sha1} from './sha1';
+
+/**
+ * Consumes service worker configuration files and processes them into control files.
+ *
+ * @experimental
+ */
+export class Generator {
+  constructor(readonly fs: Filesystem, private baseHref: string) {}
+
+  async process(config: Config): Promise<Object> {
+    const hashTable = {};
+    return {
+      configVersion: 1,
+      index: joinUrls(this.baseHref, config.index),
+      appData: config.appData,
+      assetGroups: await this.processAssetGroups(config, hashTable),
+      dataGroups: this.processDataGroups(config), hashTable,
+    };
+  }
+
+  private async processAssetGroups(config: Config, hashTable: {[file: string]: string | undefined}):
+      Promise<Object[]> {
+    const seenMap = new Set<string>();
+    return Promise.all((config.assetGroups || []).map(async(group) => {
+      const fileMatcher = globListToMatcher(group.resources.files || []);
+      const versionedMatcher = globListToMatcher(group.resources.versionedFiles || []);
+
+      const allFiles = (await this.fs.list('/'));
+
+      const versionedFiles = allFiles.filter(versionedMatcher).filter(file => !seenMap.has(file));
+      versionedFiles.forEach(file => seenMap.add(file));
+
+      const plainFiles = allFiles.filter(fileMatcher).filter(file => !seenMap.has(file));
+      plainFiles.forEach(file => seenMap.add(file));
+
+      // Add the hashes.
+      await plainFiles.reduce(async(previous, file) => {
+        await previous;
+        const hash = sha1(await this.fs.read(file));
+        hashTable[joinUrls(this.baseHref, file)] = hash;
+      }, Promise.resolve());
+
+
+      // Figure out the patterns.
+      const patterns = (group.resources.urls || [])
+                           .map(
+                               glob => glob.startsWith('/') || glob.indexOf('://') !== -1 ?
+                                   glob :
+                                   joinUrls(this.baseHref, glob))
+                           .map(glob => globToRegex(glob));
+
+      return {
+        name: group.name,
+        installMode: group.installMode || 'prefetch',
+        updateMode: group.updateMode || group.installMode || 'prefetch',
+        urls: ([] as string[])
+                  .concat(plainFiles)
+                  .concat(versionedFiles)
+                  .map(url => joinUrls(this.baseHref, url)),
+        patterns,
+      };
+    }));
+  }
+
+  private processDataGroups(config: Config): Object[] {
+    return (config.dataGroups || []).map(group => {
+      const patterns = group.urls
+                           .map(
+                               glob => glob.startsWith('/') || glob.indexOf('://') !== -1 ?
+                                   glob :
+                                   joinUrls(this.baseHref, glob))
+                           .map(glob => globToRegex(glob));
+      return {
+        name: group.name,
+        patterns,
+        strategy: group.cacheConfig.strategy || 'performance',
+        maxSize: group.cacheConfig.maxSize,
+        maxAge: parseDurationToMs(group.cacheConfig.maxAge),
+        timeoutMs: group.cacheConfig.timeout && parseDurationToMs(group.cacheConfig.timeout),
+        version: group.version !== undefined ? group.version : 1,
+      };
+    });
+  }
+}
+
+function globListToMatcher(globs: string[]): (file: string) => boolean {
+  const patterns = globs.map(pattern => {
+    if (pattern.startsWith('!')) {
+      return {
+        positive: false,
+        regex: new RegExp('^' + globToRegex(pattern.substr(1)) + '$'),
+      };
+    } else {
+      return {
+        positive: true,
+        regex: new RegExp('^' + globToRegex(pattern) + '$'),
+      };
+    }
+  });
+  return (file: string) => matches(file, patterns);
+}
+
+function matches(file: string, patterns: {positive: boolean, regex: RegExp}[]): boolean {
+  const res = patterns.reduce((isMatch, pattern) => {
+    if (pattern.positive) {
+      return isMatch || pattern.regex.test(file);
+    } else {
+      return isMatch && !pattern.regex.test(file);
+    }
+  }, false);
+  return res;
+}
+
+function joinUrls(a: string, b: string): string {
+  if (a.endsWith('/') && b.startsWith('/')) {
+    return a + b.substr(1);
+  } else if (!a.endsWith('/') && !b.startsWith('/')) {
+    return a + '/' + b;
+  }
+  return a + b;
+}

packages/service-worker/config/src/glob.ts

@@ -0,0 +1,33 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+const WILD_SINGLE = '[^\\/]+';
+const WILD_OPEN = '(?:.+\\/)?';
+
+export function globToRegex(glob: string): string {
+  const segments = glob.split('/').reverse();
+  let regex: string = '';
+  while (segments.length > 0) {
+    const segment = segments.pop() !;
+    if (segment === '**') {
+      if (segments.length > 0) {
+        regex += WILD_OPEN;
+      } else {
+        regex += '.*';
+      }
+      continue;
+    } else {
+      const processed = segment.replace(/\./g, '\\.').replace(/\*/g, WILD_SINGLE);
+      regex += processed;
+      if (segments.length > 0) {
+        regex += '\\/';
+      }
+    }
+  }
+  return regex;
+}

packages/service-worker/config/src/in.ts

@@ -0,0 +1,55 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+/**
+ * @experimental
+ */
+export type Glob = string;
+
+/**
+ * @experimental
+ */
+export type Duration = string;
+
+/**
+ * A top-level Angular Service Worker configuration object.
+ *
+ * @experimental
+ */
+export interface Config {
+  appData?: {};
+  index: string;
+  assetGroups?: AssetGroup[];
+  dataGroups?: DataGroup[];
+}
+
+/**
+ * Configuration for a particular group of assets.
+ *
+ * @experimental
+ */
+export interface AssetGroup {
+  name: string;
+  installMode?: 'prefetch'|'lazy';
+  updateMode?: 'prefetch'|'lazy';
+  resources: {files?: Glob[]; versionedFiles?: Glob[]; urls?: Glob[];};
+}
+
+/**
+ * Configuration for a particular group of dynamic URLs.
+ *
+ * @experimental
+ */
+export interface DataGroup {
+  name: string;
+  urls: Glob[];
+  version?: number;
+  cacheConfig: {
+    maxSize: number; maxAge: Duration; timeout?: Duration; strategy?: 'freshness' | 'performance';
+  };
+}

packages/service-worker/config/src/sha1.ts

@@ -0,0 +1,196 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+/**
+ * Compute the SHA1 of the given string
+ *
+ * see http://csrc.nist.gov/publications/fips/fips180-4/fips-180-4.pdf
+ *
+ * WARNING: this function has not been designed not tested with security in mind.
+ *          DO NOT USE IT IN A SECURITY SENSITIVE CONTEXT.
+ *
+ * Borrowed from @angular/compiler/src/i18n/digest.ts
+ */
+export function sha1(str: string): string {
+  const utf8 = str;
+  const words32 = stringToWords32(utf8, Endian.Big);
+  const len = utf8.length * 8;
+
+  const w = new Array(80);
+  let [a, b, c, d, e]: number[] = [0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476, 0xc3d2e1f0];
+
+  words32[len >> 5] |= 0x80 << (24 - len % 32);
+  words32[((len + 64 >> 9) << 4) + 15] = len;
+
+  for (let i = 0; i < words32.length; i += 16) {
+    const [h0, h1, h2, h3, h4]: number[] = [a, b, c, d, e];
+
+    for (let j = 0; j < 80; j++) {
+      if (j < 16) {
+        w[j] = words32[i + j];
+      } else {
+        w[j] = rol32(w[j - 3] ^ w[j - 8] ^ w[j - 14] ^ w[j - 16], 1);
+      }
+
+      const [f, k] = fk(j, b, c, d);
+      const temp = [rol32(a, 5), f, e, k, w[j]].reduce(add32);
+      [e, d, c, b, a] = [d, c, rol32(b, 30), a, temp];
+    }
+
+    [a, b, c, d, e] = [add32(a, h0), add32(b, h1), add32(c, h2), add32(d, h3), add32(e, h4)];
+  }
+
+  return byteStringToHexString(words32ToByteString([a, b, c, d, e]));
+}
+
+function add32(a: number, b: number): number {
+  return add32to64(a, b)[1];
+}
+
+function add32to64(a: number, b: number): [number, number] {
+  const low = (a & 0xffff) + (b & 0xffff);
+  const high = (a >>> 16) + (b >>> 16) + (low >>> 16);
+  return [high >>> 16, (high << 16) | (low & 0xffff)];
+}
+
+function add64([ah, al]: [number, number], [bh, bl]: [number, number]): [number, number] {
+  const [carry, l] = add32to64(al, bl);
+  const h = add32(add32(ah, bh), carry);
+  return [h, l];
+}
+
+function sub32(a: number, b: number): number {
+  const low = (a & 0xffff) - (b & 0xffff);
+  const high = (a >> 16) - (b >> 16) + (low >> 16);
+  return (high << 16) | (low & 0xffff);
+}
+
+// Rotate a 32b number left `count` position
+function rol32(a: number, count: number): number {
+  return (a << count) | (a >>> (32 - count));
+}
+
+// Rotate a 64b number left `count` position
+function rol64([hi, lo]: [number, number], count: number): [number, number] {
+  const h = (hi << count) | (lo >>> (32 - count));
+  const l = (lo << count) | (hi >>> (32 - count));
+  return [h, l];
+}
+
+enum Endian {
+  Little,
+  Big,
+}
+
+function fk(index: number, b: number, c: number, d: number): [number, number] {
+  if (index < 20) {
+    return [(b & c) | (~b & d), 0x5a827999];
+  }
+
+  if (index < 40) {
+    return [b ^ c ^ d, 0x6ed9eba1];
+  }
+
+  if (index < 60) {
+    return [(b & c) | (b & d) | (c & d), 0x8f1bbcdc];
+  }
+
+  return [b ^ c ^ d, 0xca62c1d6];
+}
+
+
+function stringToWords32(str: string, endian: Endian): number[] {
+  const words32 = Array((str.length + 3) >>> 2);
+
+  for (let i = 0; i < words32.length; i++) {
+    words32[i] = wordAt(str, i * 4, endian);
+  }
+
+  return words32;
+}
+
+function byteAt(str: string, index: number): number {
+  return index >= str.length ? 0 : str.charCodeAt(index) & 0xff;
+}
+
+function wordAt(str: string, index: number, endian: Endian): number {
+  let word = 0;
+  if (endian === Endian.Big) {
+    for (let i = 0; i < 4; i++) {
+      word += byteAt(str, index + i) << (24 - 8 * i);
+    }
+  } else {
+    for (let i = 0; i < 4; i++) {
+      word += byteAt(str, index + i) << 8 * i;
+    }
+  }
+  return word;
+}
+
+function words32ToByteString(words32: number[]): string {
+  return words32.reduce((str, word) => str + word32ToByteString(word), '');
+}
+
+function word32ToByteString(word: number): string {
+  let str = '';
+  for (let i = 0; i < 4; i++) {
+    str += String.fromCharCode((word >>> 8 * (3 - i)) & 0xff);
+  }
+  return str;
+}
+
+function byteStringToHexString(str: string): string {
+  let hex: string = '';
+  for (let i = 0; i < str.length; i++) {
+    const b = byteAt(str, i);
+    hex += (b >>> 4).toString(16) + (b & 0x0f).toString(16);
+  }
+  return hex.toLowerCase();
+}
+
+// based on http://www.danvk.org/hex2dec.html (JS can not handle more than 56b)
+function byteStringToDecString(str: string): string {
+  let decimal = '';
+  let toThePower = '1';
+
+  for (let i = str.length - 1; i >= 0; i--) {
+    decimal = addBigInt(decimal, numberTimesBigInt(byteAt(str, i), toThePower));
+    toThePower = numberTimesBigInt(256, toThePower);
+  }
+
+  return decimal.split('').reverse().join('');
+}
+
+// x and y decimal, lowest significant digit first
+function addBigInt(x: string, y: string): string {
+  let sum = '';
+  const len = Math.max(x.length, y.length);
+  for (let i = 0, carry = 0; i < len || carry; i++) {
+    const tmpSum = carry + +(x[i] || 0) + +(y[i] || 0);
+    if (tmpSum >= 10) {
+      carry = 1;
+      sum += tmpSum - 10;
+    } else {
+      carry = 0;
+      sum += tmpSum;
+    }
+  }
+
+  return sum;
+}
+
+
+function numberTimesBigInt(num: number, b: string): string {
+  let product = '';
+  let bToThePower = b;
+  for (; num !== 0; num = num >>> 1) {
+    if (num & 1) product = addBigInt(product, bToThePower);
+    bToThePower = addBigInt(bToThePower, bToThePower);
+  }
+  return product;
+}

packages/service-worker/config/test/generator_spec.ts

@@ -0,0 +1,86 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {Generator} from '../src/generator';
+import {MockFilesystem} from '../testing/mock';
+
+export function main() {
+  describe('Generator', () => {
+    it('generates a correct config', (done: DoneFn) => {
+      const fs = new MockFilesystem({
+        '/index.html': 'This is a test',
+        '/foo/test.html': 'Another test',
+        '/ignored/x.html': 'should be ignored',
+      });
+      const gen = new Generator(fs, '/test');
+      const res = gen.process({
+        index: '/index.html',
+        appData: {
+          test: true,
+        },
+        assetGroups: [{
+          name: 'test',
+          resources: {
+            files: [
+              '/**/*.html', '!/ignored/**',
+              //                '/*.html',
+            ],
+            versionedFiles: [],
+            urls: [
+              '/absolute/**',
+              'relative/*.txt',
+            ]
+          }
+        }],
+        dataGroups: [{
+          name: 'other',
+          urls: [
+            '/api/**',
+            'relapi/**',
+          ],
+          cacheConfig: {
+            maxSize: 100,
+            maxAge: '3d',
+            timeout: '1m',
+          }
+        }]
+      });
+      res.then(config => {
+           expect(config).toEqual({
+             'configVersion': 1,
+             'index': '/test/index.html',
+             'appData': {
+               'test': true,
+             },
+             'assetGroups': [{
+               'name': 'test',
+               'installMode': 'prefetch',
+               'updateMode': 'prefetch',
+               'urls': ['/test/index.html', '/test/foo/test.html'],
+               'patterns': ['\\/absolute\\/.*', '\\/test\\/relative\\/[^\\/]+\\.txt']
+             }],
+             'dataGroups': [{
+               'name': 'other',
+               'patterns': ['\\/api\\/.*', '\\/test\\/relapi\\/.*'],
+               'strategy': 'performance',
+               'maxSize': 100,
+               'maxAge': 259200000,
+               'timeoutMs': 60000,
+               'version': 1,
+             }],
+             'hashTable': {
+               '/test/index.html': 'a54d88e06612d820bc3be72877c74f257b561b19',
+               '/test/foo/test.html': '18f6f8eb7b1c23d2bb61bff028b83d867a9e4643'
+             }
+           });
+           done();
+         })
+          .catch(err => done.fail(err));
+    });
+  });
+}

packages/service-worker/config/testing/mock.ts

@@ -0,0 +1,25 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {Filesystem} from '../src/filesystem';
+
+export class MockFilesystem implements Filesystem {
+  private files = new Map<string, string>();
+
+  constructor(files: {[name: string]: string | undefined}) {
+    Object.keys(files).forEach(path => this.files.set(path, files[path] !));
+  }
+
+  async list(dir: string): Promise<string[]> {
+    return Array.from(this.files.keys()).filter(path => path.startsWith(dir));
+  }
+
+  async read(path: string): Promise<string> { return this.files.get(path) !; }
+
+  async write(path: string, contents: string): Promise<void> { this.files.set(path, contents); }
+}

packages/service-worker/config/tsconfig-build.json

@@ -0,0 +1,23 @@
+{
+  "extends": "../tsconfig-build.json",
+
+  "compilerOptions": {
+    "baseUrl": ".",
+    "rootDir": "../",
+    "paths": {
+      "@angular/core": ["../../../dist/packages/core"]
+    },
+    "outDir": "../../../dist/packages/service-worker"
+  },
+
+  "files": [
+    "public_api.ts"
+  ],
+
+  "angularCompilerOptions": {
+    "annotateForClosureCompiler": true,
+    "strictMetadataEmit": false,
+    "flatModuleOutFile": "config.js",
+    "flatModuleId": "@angular/service-worker/config"
+  }
+}