build: add size-tracking bazel test (#30070)

Introduces a new Bazel test that allows us to inspect
what source-files contribute to a given bundled file
and how much bytes they contribute to the bundle size.

Additionally the size-tracking rule groups the size
data by directories. This allows us to compare size
changes in the scope of directories. e.g. a lot of
files in a directory could increase slightly in size, but
in the directory scope the size change could be significant
and needs to be reported by the test target.

Resolves FW-1278

PR Close #30070

tools/size-tracking/BUILD.bazel

@@ -0,0 +1,35 @@
+load("//tools:defaults.bzl", "jasmine_node_test", "ts_library")
+
+package(default_visibility = ["//visibility:public"])
+
+ts_library(
+    name = "size-tracking",
+    srcs = glob(
+        ["**/*.ts"],
+        exclude = ["**/*_spec.ts"],
+    ),
+    tsconfig = "//tools:tsconfig.json",
+    deps = [
+        "@npm//@types/node",
+        "@npm//@types/source-map",
+    ],
+)
+
+ts_library(
+    name = "test_lib",
+    testonly = True,
+    srcs = glob(["**/*_spec.ts"]),
+    deps = [
+        ":size-tracking",
+        "@npm//@types/source-map",
+    ],
+)
+
+jasmine_node_test(
+    name = "test",
+    data = [],
+    deps = [
+        ":test_lib",
+        "@npm//source-map",
+    ],
+)

tools/size-tracking/file_size_compare.ts

@@ -0,0 +1,96 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {DirectorySizeEntry, FileSizeData, getChildEntryNames} from './file_size_data';
+
+export interface SizeDifference {
+  filePath?: string;
+  message: string;
+}
+
+/** Compares two file size data objects and returns an array of size differences. */
+export function compareFileSizeData(
+    actual: FileSizeData, expected: FileSizeData, threshold: number) {
+  const diffs: SizeDifference[] = compareSizeEntry(actual.files, expected.files, '/', threshold);
+  const unmappedBytesDiff = getDifferencePercentage(actual.unmapped, expected.unmapped);
+  if (unmappedBytesDiff > threshold) {
+    diffs.push({
+      message: `Unmapped bytes differ by ${unmappedBytesDiff.toFixed(2)}% from ` +
+          `the expected size (actual = ${actual.unmapped}, expected = ${expected.unmapped})`
+    });
+  }
+  return diffs;
+}
+
+/** Compares two file size entries and returns an array of size differences. */
+function compareSizeEntry(
+    actual: DirectorySizeEntry | number, expected: DirectorySizeEntry | number, filePath: string,
+    threshold: number) {
+  if (typeof actual !== 'number' && typeof expected !== 'number') {
+    return compareDirectorySizeEntry(
+        <DirectorySizeEntry>actual, <DirectorySizeEntry>expected, filePath, threshold);
+  } else {
+    return compareActualSizeToExpected(<number>actual, <number>expected, filePath, threshold);
+  }
+}
+
+/**
+ * Compares two size numbers and returns a size difference when the percentage difference
+ * exceeds the specified threshold.
+ */
+function compareActualSizeToExpected(
+    actualSize: number, expectedSize: number, filePath: string,
+    threshold: number): SizeDifference[] {
+  const diffPercentage = getDifferencePercentage(actualSize, expectedSize);
+  if (diffPercentage > threshold) {
+    return [{
+      filePath: filePath,
+      message: `Differs by ${diffPercentage.toFixed(2)}% from the expected size ` +
+          `(actual = ${actualSize}, expected = ${expectedSize})`
+    }];
+  }
+  return [];
+}
+
+/**
+ * Compares two size directory size entries and returns an array of found size
+ * differences within that directory.
+ */
+function compareDirectorySizeEntry(
+    actual: DirectorySizeEntry, expected: DirectorySizeEntry, filePath: string,
+    threshold: number): SizeDifference[] {
+  const diffs: SizeDifference[] =
+      [...compareActualSizeToExpected(actual.size, expected.size, filePath, threshold)];
+
+  getChildEntryNames(expected).forEach(childName => {
+    if (actual[childName] === undefined) {
+      diffs.push(
+          {filePath: filePath + childName, message: 'Expected file/directory is not included.'});
+      return;
+    }
+
+    diffs.push(...compareSizeEntry(
+        actual[childName], expected[childName], filePath + childName, threshold));
+  });
+
+  getChildEntryNames(actual).forEach(childName => {
+    if (expected[childName] === undefined) {
+      diffs.push({
+        filePath: filePath + childName,
+        message: 'Unexpected file/directory included (not part of golden).'
+      });
+    }
+  });
+
+  return diffs;
+}
+
+/** Gets the difference of the two size values in percentage. */
+function getDifferencePercentage(actualSize: number, expectedSize: number) {
+  return (Math.abs(actualSize - expectedSize) / ((expectedSize + actualSize) / 2)) * 100;
+}

tools/size-tracking/file_size_compare_spec.ts

@@ -0,0 +1,92 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {compareFileSizeData} from './file_size_compare';
+
+describe('file size compare', () => {
+
+  it('should report if size entry differ by more than the specified threshold', () => {
+    const diffs = compareFileSizeData(
+        {
+          unmapped: 0,
+          files: {
+            size: 50,
+            'a.ts': 50,
+          }
+        },
+        {
+          unmapped: 0,
+          files: {
+            size: 75,
+            'a.ts': 75,
+          }
+        },
+        0);
+
+    expect(diffs.length).toBe(2);
+    expect(diffs[0].filePath).toBe('/');
+    expect(diffs[0].message).toMatch(/40.00% from the expected size/);
+    expect(diffs[1].filePath).toBe('/a.ts');
+    expect(diffs[1].message).toMatch(/40.00% from the expected size/);
+  });
+
+  it('should not report if size percentage difference does not exceed threshold', () => {
+    const diffs = compareFileSizeData(
+        {
+          unmapped: 0,
+          files: {
+            size: 50,
+            'a.ts': 50,
+          }
+        },
+        {
+          unmapped: 0,
+          files: {
+            size: 75,
+            'a.ts': 75,
+          }
+        },
+        40);
+
+    expect(diffs.length).toBe(0);
+  });
+
+
+  it('should report if expected file size data misses a file size entry', () => {
+    const diffs = compareFileSizeData(
+        {
+          unmapped: 0,
+          files: {
+            size: 101,
+            'a.ts': 100,
+            'b.ts': 1,
+          }
+        },
+        {unmapped: 0, files: {size: 100, 'a.ts': 100}}, 1);
+
+    expect(diffs.length).toBe(1);
+    expect(diffs[0].filePath).toBe('/b.ts');
+    expect(diffs[0].message).toMatch(/Unexpected file.*not part of golden./);
+  });
+
+  it('should report if actual file size data misses an expected file size entry', () => {
+    const diffs = compareFileSizeData(
+        {
+          unmapped: 0,
+          files: {
+            size: 100,
+            'a.ts': 100,
+          }
+        },
+        {unmapped: 0, files: {size: 101, 'a.ts': 100, 'b.ts': 1}}, 1);
+
+    expect(diffs.length).toBe(1);
+    expect(diffs[0].filePath).toBe('/b.ts');
+    expect(diffs[0].message).toMatch(/Expected file.*not included./);
+  });
+});

tools/size-tracking/file_size_data.ts

@@ -0,0 +1,75 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+export interface DirectorySizeEntry {
+  size: number;
+  [filePath: string]: DirectorySizeEntry|number;
+}
+
+export interface FileSizeData {
+  unmapped: number;
+  files: DirectorySizeEntry;
+}
+
+/** Returns a new file size data sorted by keys in ascending alphabetical order. */
+export function sortFileSizeData({unmapped, files}: FileSizeData): FileSizeData {
+  return {unmapped, files: _sortDirectorySizeEntryObject(files)};
+}
+
+/** Gets the name of all child size entries of the specified one. */
+export function getChildEntryNames(entry: DirectorySizeEntry): string[] {
+  // The "size" property is reserved for the stored size value.
+  return Object.keys(entry).filter(key => key !== 'size');
+}
+
+/**
+ * Returns the first size-entry that has multiple children. This is also known as
+ * the omitting of the common path prefix.
+ * */
+export function omitCommonPathPrefix(entry: DirectorySizeEntry): DirectorySizeEntry {
+  let current: DirectorySizeEntry = entry;
+  while (getChildEntryNames(current).length === 1) {
+    const newChild = current[getChildEntryNames(current)[0]];
+    // Only omit the current node if it is a size entry. In case the new
+    // child is a holding a number, then this is a file and we don'twant
+    // to incorrectly omit the leaf file entries.
+    if (typeof newChild === 'number') {
+      break;
+    }
+    current = newChild;
+  }
+  return current;
+}
+
+function _sortDirectorySizeEntryObject(oldObject: DirectorySizeEntry): DirectorySizeEntry {
+  return Object.keys(oldObject)
+      .sort(_sortSizeEntryKeys)
+      .reduce(
+          (result, key) => {
+            if (typeof oldObject[key] === 'number') {
+              result[key] = oldObject[key];
+            } else {
+              result[key] = _sortDirectorySizeEntryObject(oldObject[key] as DirectorySizeEntry);
+            }
+            return result;
+          },
+          {} as DirectorySizeEntry);
+}
+
+function _sortSizeEntryKeys(a: string, b: string) {
+  // The "size" property should always be the first item in the size entry.
+  // This makes it easier to inspect the size of an entry in the golden.
+  if (a === 'size') {
+    return -1;
+  } else if (a < b) {
+    return -1;
+  } else if (a > b) {
+    return 1;
+  }
+  return 0;
+}

tools/size-tracking/file_size_data_spec.ts

@@ -0,0 +1,64 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {FileSizeData, omitCommonPathPrefix, sortFileSizeData} from './file_size_data';
+
+describe('file size data', () => {
+  it('should be able to properly omit the common path prefix', () => {
+    const data: FileSizeData = {
+      unmapped: 0,
+      files: {
+        size: 3,
+        'parent/': {
+          size: 3,
+          'parent2/': {
+            size: 3,
+            'a/': {
+              size: 3,
+              'file.ts': 3,
+            },
+            'b/': {
+              size: 0,
+            }
+          }
+        }
+      }
+    };
+
+    expect(omitCommonPathPrefix(data.files)).toEqual({
+      size: 3,
+      'a/': {
+        size: 3,
+        'file.ts': 3,
+      },
+      'b/': {
+        size: 0,
+      }
+    });
+  });
+
+  it('should be able to properly sort file size data in alphabetical order', () => {
+    const data: FileSizeData = {
+      unmapped: 0,
+      files: {
+        size: 7,
+        'b/': {'c.ts': 3, 'a.ts': 3, size: 6},
+        'a/': {'nested/': {size: 1, 'a.ts': 1}, size: 1},
+      }
+    };
+
+    expect(sortFileSizeData(data)).toEqual({
+      unmapped: 0,
+      files: {
+        size: 7,
+        'a/': {size: 1, 'nested/': {size: 1, 'a.ts': 1}},
+        'b/': {size: 6, 'a.ts': 3, 'c.ts': 3},
+      },
+    });
+  });
+});

tools/size-tracking/index.bzl

@@ -0,0 +1,39 @@
+# Copyright Google Inc. All Rights Reserved.
+#
+# Use of this source code is governed by an MIT-style license that can be
+# found in the LICENSE file at https://angular.io/license
+
+load("@build_bazel_rules_nodejs//:defs.bzl", "nodejs_binary", "nodejs_test")
+
+"""
+  Macro that can be used to track the size of a given input file by inspecting
+  the corresponding source map. A golden file is used to compare the current
+  file size data against previously approved file size data
+"""
+
+def js_size_tracking_test(name, src, sourceMap, goldenFile, diffThreshold, data = [], **kwargs):
+    all_data = data + [
+        "//tools/size-tracking",
+        "@npm//source-map",
+        "@npm//chalk",
+    ]
+    entry_point = "angular/tools/size-tracking/index.js"
+
+    nodejs_test(
+        name = name,
+        data = all_data,
+        entry_point = entry_point,
+        configuration_env_vars = ["compile"],
+        templated_args = [src, sourceMap, goldenFile, "%d" % diffThreshold, "false"],
+        **kwargs
+    )
+
+    nodejs_binary(
+        name = "%s.accept" % name,
+        testonly = True,
+        data = all_data,
+        entry_point = entry_point,
+        configuration_env_vars = ["compile"],
+        templated_args = [src, sourceMap, goldenFile, "%d" % diffThreshold, "true"],
+        **kwargs
+    )

tools/size-tracking/index.ts

@@ -0,0 +1,56 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {readFileSync, writeFileSync} from 'fs';
+import {SizeTracker} from './size_tracker';
+import chalk from 'chalk';
+import {compareFileSizeData} from './file_size_compare';
+import {FileSizeData} from './file_size_data';
+
+if (require.main === module) {
+  const [filePath, sourceMapPath, goldenPath, thresholdArg, writeGoldenArg] = process.argv.slice(2);
+  const status = main(
+      require.resolve(filePath), require.resolve(sourceMapPath), require.resolve(goldenPath),
+      writeGoldenArg === 'true', parseInt(thresholdArg));
+
+  process.exit(status ? 0 : 1);
+}
+
+export function main(
+    filePath: string, sourceMapPath: string, goldenSizeMapPath: string, writeGolden: boolean,
+    diffThreshold: number): boolean {
+  const {sizeResult} = new SizeTracker(filePath, sourceMapPath);
+
+  if (writeGolden) {
+    writeFileSync(goldenSizeMapPath, JSON.stringify(sizeResult, null, 2));
+    console.error(chalk.green(`Updated golden size data in ${goldenSizeMapPath}`));
+    return;
+  }
+
+  const expectedSizeData = <FileSizeData>JSON.parse(readFileSync(goldenSizeMapPath, 'utf8'));
+  const differences = compareFileSizeData(sizeResult, expectedSizeData, diffThreshold);
+
+  if (!differences.length) {
+    return true;
+  }
+
+  console.error(
+      `Computed file size data does not match golden size data. ` +
+      `The following differences were found:\n`);
+  differences.forEach(({filePath, message}) => {
+    const failurePrefix = filePath ? `"${filePath}": ` : '';
+    console.error(chalk.red(`    ${failurePrefix}${message}`));
+  });
+
+  const compile = process.env['compile'];
+  const defineFlag = (compile !== 'legacy') ? `--define=compile=${compile} ` : '';
+  const bazelTargetName = process.env['TEST_TARGET'];
+
+  console.error(`\nThe golden file can be updated with the following command:`);
+  console.error(`    yarn bazel run ${defineFlag}${bazelTargetName}.accept`);
+}

tools/size-tracking/size_tracker.ts

@@ -0,0 +1,104 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {readFileSync} from 'fs';
+import {SourceMapConsumer} from 'source-map';
+
+import {DirectorySizeEntry, FileSizeData, omitCommonPathPrefix, sortFileSizeData} from './file_size_data';
+
+export class SizeTracker {
+  private fileContent: string;
+  private consumer: SourceMapConsumer;
+
+  /**
+   * Retraced size result that can be used to inspect where bytes in the input file
+   * originated from and how much each file contributes to the input file.
+   */
+  readonly sizeResult: FileSizeData;
+
+  constructor(private filePath: string, private sourceMapPath: string) {
+    this.fileContent = readFileSync(filePath, 'utf8');
+    this.consumer = new SourceMapConsumer(JSON.parse(readFileSync(sourceMapPath, 'utf8')));
+    this.sizeResult = this._computeSizeResult();
+  }
+
+  /**
+   * Computes the file size data by analyzing the input file through the specified
+   * source-map.
+   */
+  private _computeSizeResult(): FileSizeData {
+    const lines = this.fileContent.split(/(\r?\n)/);
+    const result: FileSizeData = {
+      unmapped: 0,
+      files: {size: 0},
+    };
+
+    // Walk through the columns for each line in the input file and find the
+    // origin source-file of the given character. This allows us to inspect
+    // how the given input file is composed and how much each individual file
+    // contributes to the overall bundle file.
+    for (let lineIdx = 0; lineIdx < lines.length; lineIdx++) {
+      const lineText = lines[lineIdx];
+      for (let colIdx = 0; colIdx < lineText.length; colIdx++) {
+        // Note that the "originalPositionFor" line number is one-based.
+        let {source} = this.consumer.originalPositionFor({line: lineIdx + 1, column: colIdx});
+
+        // Increase the amount of total bytes.
+        result.files.size += 1;
+
+        if (!source) {
+          result.unmapped += 1;
+          continue;
+        }
+
+        const pathSegments = this._resolveMappedPath(source).split('/');
+        let currentEntry = result.files;
+
+        // Walk through each path segment and update the size entries with
+        // new size. This makes it possibly to create na hierarchical tree
+        // that matches the actual file system.
+        pathSegments.forEach((segmentName, index) => {
+          // The last segment always refers to a file and we therefore can
+          // store the size verbatim as property value.
+          if (index === pathSegments.length - 1) {
+            currentEntry[segmentName] = (<number>currentEntry[segmentName] || 0) + 1;
+          } else {
+            // Append a trailing slash to the segment so that it
+            // is clear that this size entry represents a folder.
+            segmentName = `${segmentName}/`;
+            const newEntry = <DirectorySizeEntry>currentEntry[segmentName] || {size: 0};
+            newEntry.size += 1;
+            currentEntry = currentEntry[segmentName] = newEntry;
+          }
+        });
+      }
+    }
+
+    // Omit size entries which are not needed and just bloat up the file
+    // size data. e.g. if all paths start with "../../", we want to omit
+    // this prefix to make the size data less confusing.
+    result.files = omitCommonPathPrefix(result.files);
+
+    return sortFileSizeData(result);
+  }
+
+  private _resolveMappedPath(filePath: string): string {
+    // We only want to store POSIX-like paths in order to avoid path
+    // separator failures when running the golden tests on Windows.
+    filePath = filePath.replace(/\\/g, '/');
+
+    // Workaround for https://github.com/angular/angular/issues/30060
+    if (process.env['BAZEL_TARGET'].includes('test/bundling/core_all:size_test')) {
+      return filePath.replace(/^(\.\.\/)+external/, 'external')
+          .replace(/^(\.\.\/)+packages\/core\//, '@angular/core/')
+          .replace(/^(\.\.\/){3}/, '@angular/core/');
+    }
+
+    return filePath;
+  }
+}

tools/size-tracking/size_tracking_spec.ts

@@ -0,0 +1,111 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {writeFileSync} from 'fs';
+import {join} from 'path';
+import {SourceMapGenerator} from 'source-map';
+
+import {SizeTracker} from './size_tracker';
+
+const testTempDir = process.env['TEST_TMPDIR'] !;
+
+describe('size tracking', () => {
+  let generator: SourceMapGenerator;
+
+  beforeEach(() => { generator = new SourceMapGenerator(); });
+
+  function writeFile(filePath: string, content: string): string {
+    const tmpFilePath = join(testTempDir, filePath);
+    writeFileSync(tmpFilePath, content);
+    return tmpFilePath;
+  }
+
+  it('should keep track of unmapped bytes in the file', () => {
+    generator.addMapping({
+      generated: {line: 1, column: 1},
+      original: {line: 1, column: 1},
+      source: './origin-a.ts',
+    });
+
+    // A => origin-a (2 bytes), U => unmapped (1 byte)
+    const mapPath = writeFile('/test.map', generator.toString());
+    const inputPath = writeFile('/test.js', `UAA`);
+
+    const {sizeResult} = new SizeTracker(inputPath, mapPath);
+
+    expect(sizeResult.unmapped).toBe(1);
+    expect(sizeResult.files).toEqual({
+      size: 3,
+      'origin-a.ts': 2,
+    });
+  });
+
+  it('should properly combine mapped characters from same source', () => {
+    generator.addMapping(
+        {generated: {line: 1, column: 0}, original: {line: 1, column: 0}, source: './origin-a.ts'});
+
+    generator.addMapping(
+        {generated: {line: 1, column: 1}, original: {line: 1, column: 0}, source: './origin-b.ts'});
+
+    generator.addMapping({
+      generated: {line: 1, column: 2},
+      original: {line: 10, column: 0},
+      source: './origin-a.ts'
+    });
+
+    // A => origin-a (1 byte), B => origin-b (two bytes)
+    const mapPath = writeFile('/test.map', generator.toString());
+    const inputPath = writeFile('/test.js', `ABB`);
+
+    const {sizeResult} = new SizeTracker(inputPath, mapPath);
+
+    expect(sizeResult.unmapped).toBe(0);
+    expect(sizeResult.files).toEqual({
+      size: 3,
+      'origin-a.ts': 2,
+      'origin-b.ts': 1,
+    });
+  });
+
+  it('should keep track of summed-up byte sizes for directories', () => {
+    generator.addMapping({
+      generated: {line: 1, column: 0},
+      original: {line: 1, column: 0},
+      source: '@angular/core/render3/a.ts'
+    });
+
+    generator.addMapping({
+      generated: {line: 1, column: 2},
+      original: {line: 1, column: 0},
+      source: '@angular/core/render3/b.ts'
+    });
+
+    generator.addMapping({
+      generated: {line: 1, column: 3},
+      original: {line: 1, column: 0},
+      source: '@angular/core/c.ts'
+    });
+
+    // A => render3/a.ts (2 bytes), B => render3/b.ts (1 byte), C => c.ts (1 byte)
+    const mapPath = writeFile('/test.map', generator.toString());
+    const inputPath = writeFile('/test.js', `AABC`);
+
+    const {sizeResult} = new SizeTracker(inputPath, mapPath);
+
+    expect(sizeResult.unmapped).toBe(0);
+    expect(sizeResult.files).toEqual({
+      size: 4,
+      'render3/': {
+        size: 3,
+        'a.ts': 2,
+        'b.ts': 1,
+      },
+      'c.ts': 1,
+    });
+  });
+});