refactor(ivy): implement a virtual file-system layer in ngtsc + ngcc (#30921)

To improve cross platform support, all file access (and path manipulation)
is now done through a well known interface (`FileSystem`).

For testing a number of `MockFileSystem` implementations are provided.
These provide an in-memory file-system which emulates operating systems
like OS/X, Unix and Windows.

The current file system is always available via the static method,
`FileSystem.getFileSystem()`. This is also used by a number of static
methods on `AbsoluteFsPath` and `PathSegment`, to avoid having to pass
`FileSystem` objects around all the time. The result of this is that one
must be careful to ensure that the file-system has been initialized before
using any of these static methods. To prevent this happening accidentally
the current file system always starts out as an instance of `InvalidFileSystem`,
which will throw an error if any of its methods are called.

You can set the current file-system by calling `FileSystem.setFileSystem()`.
During testing you can call the helper function `initMockFileSystem(os)`
which takes a string name of the OS to emulate, and will also monkey-patch
aspects of the TypeScript library to ensure that TS is also using the
current file-system.

Finally there is the `NgtscCompilerHost` to be used for any TypeScript
compilation, which uses a given file-system.

All tests that interact with the file-system should be tested against each
of the mock file-systems. A series of helpers have been provided to support
such tests:

* `runInEachFileSystem()` - wrap your tests in this helper to run all the
wrapped tests in each of the mock file-systems.
* `addTestFilesToFileSystem()` - use this to add files and their contents
to the mock file system for testing.
* `loadTestFilesFromDisk()` - use this to load a mirror image of files on
disk into the in-memory mock file-system.
* `loadFakeCore()` - use this to load a fake version of `@angular/core`
into the mock file-system.

All ngcc and ngtsc source and tests now use this virtual file-system setup.

PR Close #30921

packages/compiler-cli/src/ngtsc/file_system/testing/BUILD.bazel

@@ -0,0 +1,16 @@
+load("//tools:defaults.bzl", "ts_library")
+
+package(default_visibility = ["//visibility:public"])
+
+ts_library(
+    name = "testing",
+    testonly = True,
+    srcs = glob([
+        "**/*.ts",
+    ]),
+    deps = [
+        "//packages:types",
+        "//packages/compiler-cli/src/ngtsc/file_system",
+        "@npm//typescript",
+    ],
+)

packages/compiler-cli/src/ngtsc/file_system/testing/index.ts

@@ -0,0 +1,13 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+export {Folder, MockFileSystem} from './src/mock_file_system';
+export {MockFileSystemNative} from './src/mock_file_system_native';
+export {MockFileSystemPosix} from './src/mock_file_system_posix';
+export {MockFileSystemWindows} from './src/mock_file_system_windows';
+export {TestFile, initMockFileSystem, runInEachFileSystem} from './src/test_helper';

packages/compiler-cli/src/ngtsc/file_system/testing/src/mock_file_system.ts

@@ -0,0 +1,239 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+
+import {basename, dirname, resolve} from '../../src/helpers';
+import {AbsoluteFsPath, FileStats, FileSystem, PathSegment, PathString} from '../../src/types';
+
+/**
+ * An in-memory file system that can be used in unit tests.
+ */
+export abstract class MockFileSystem implements FileSystem {
+  private _fileTree: Folder = {};
+  private _cwd: AbsoluteFsPath;
+
+
+  constructor(private _isCaseSensitive = false, cwd: AbsoluteFsPath = '/' as AbsoluteFsPath) {
+    this._cwd = this.normalize(cwd);
+  }
+
+  isCaseSensitive() { return this._isCaseSensitive; }
+
+  exists(path: AbsoluteFsPath): boolean { return this.findFromPath(path).entity !== null; }
+
+  readFile(path: AbsoluteFsPath): string {
+    const {entity} = this.findFromPath(path);
+    if (isFile(entity)) {
+      return entity;
+    } else {
+      throw new MockFileSystemError('ENOENT', path, `File "${path}" does not exist.`);
+    }
+  }
+
+  writeFile(path: AbsoluteFsPath, data: string): void {
+    const [folderPath, basename] = this.splitIntoFolderAndFile(path);
+    const {entity} = this.findFromPath(folderPath);
+    if (entity === null || !isFolder(entity)) {
+      throw new MockFileSystemError(
+          'ENOENT', path, `Unable to write file "${path}". The containing folder does not exist.`);
+    }
+    entity[basename] = data;
+  }
+
+  symlink(target: AbsoluteFsPath, path: AbsoluteFsPath): void {
+    const [folderPath, basename] = this.splitIntoFolderAndFile(path);
+    const {entity} = this.findFromPath(folderPath);
+    if (entity === null || !isFolder(entity)) {
+      throw new MockFileSystemError(
+          'ENOENT', path,
+          `Unable to create symlink at "${path}". The containing folder does not exist.`);
+    }
+    entity[basename] = new SymLink(target);
+  }
+
+  readdir(path: AbsoluteFsPath): PathSegment[] {
+    const {entity} = this.findFromPath(path);
+    if (entity === null) {
+      throw new MockFileSystemError(
+          'ENOENT', path, `Unable to read directory "${path}". It does not exist.`);
+    }
+    if (isFile(entity)) {
+      throw new MockFileSystemError(
+          'ENOTDIR', path, `Unable to read directory "${path}". It is a file.`);
+    }
+    return Object.keys(entity) as PathSegment[];
+  }
+
+  lstat(path: AbsoluteFsPath): FileStats {
+    const {entity} = this.findFromPath(path);
+    if (entity === null) {
+      throw new MockFileSystemError('ENOENT', path, `File "${path}" does not exist.`);
+    }
+    return new MockFileStats(entity);
+  }
+
+  stat(path: AbsoluteFsPath): FileStats {
+    const {entity} = this.findFromPath(path, {followSymLinks: true});
+    if (entity === null) {
+      throw new MockFileSystemError('ENOENT', path, `File "${path}" does not exist.`);
+    }
+    return new MockFileStats(entity);
+  }
+
+  copyFile(from: AbsoluteFsPath, to: AbsoluteFsPath): void {
+    this.writeFile(to, this.readFile(from));
+  }
+
+  moveFile(from: AbsoluteFsPath, to: AbsoluteFsPath): void {
+    this.writeFile(to, this.readFile(from));
+    const result = this.findFromPath(dirname(from));
+    const folder = result.entity as Folder;
+    const name = basename(from);
+    delete folder[name];
+  }
+
+  mkdir(path: AbsoluteFsPath): void { this.ensureFolders(this._fileTree, this.splitPath(path)); }
+
+  ensureDir(path: AbsoluteFsPath): void {
+    this.ensureFolders(this._fileTree, this.splitPath(path));
+  }
+
+  isRoot(path: AbsoluteFsPath): boolean { return this.dirname(path) === path; }
+
+  extname(path: AbsoluteFsPath|PathSegment): string {
+    const match = /.+(\.[^.]*)$/.exec(path);
+    return match !== null ? match[1] : '';
+  }
+
+  realpath(filePath: AbsoluteFsPath): AbsoluteFsPath {
+    const result = this.findFromPath(filePath, {followSymLinks: true});
+    if (result.entity === null) {
+      throw new MockFileSystemError(
+          'ENOENT', filePath, `Unable to find the real path of "${filePath}". It does not exist.`);
+    } else {
+      return result.path;
+    }
+  }
+
+  pwd(): AbsoluteFsPath { return this._cwd; }
+
+  getDefaultLibLocation(): AbsoluteFsPath { return this.resolve('node_modules/typescript/lib'); }
+
+  abstract resolve(...paths: string[]): AbsoluteFsPath;
+  abstract dirname<T extends string>(file: T): T;
+  abstract join<T extends string>(basePath: T, ...paths: string[]): T;
+  abstract relative<T extends PathString>(from: T, to: T): PathSegment;
+  abstract basename(filePath: string, extension?: string): PathSegment;
+  abstract isRooted(path: string): boolean;
+  abstract normalize<T extends PathString>(path: T): T;
+  protected abstract splitPath<T extends PathString>(path: T): string[];
+
+  dump(): Folder { return cloneFolder(this._fileTree); }
+  init(folder: Folder): void { this._fileTree = cloneFolder(folder); }
+
+  protected findFromPath(path: AbsoluteFsPath, options?: {followSymLinks: boolean}): FindResult {
+    const followSymLinks = !!options && options.followSymLinks;
+    const segments = this.splitPath(path);
+    if (segments.length > 1 && segments[segments.length - 1] === '') {
+      // Remove a trailing slash (unless the path was only `/`)
+      segments.pop();
+    }
+    // Convert the root folder to a canonical empty string `""` (on Windows it would be `C:`).
+    segments[0] = '';
+    let current: Entity|null = this._fileTree;
+    while (segments.length) {
+      current = current[segments.shift() !];
+      if (current === undefined) {
+        return {path, entity: null};
+      }
+      if (segments.length > 0 && (!isFolder(current))) {
+        current = null;
+        break;
+      }
+      if (isFile(current)) {
+        break;
+      }
+      if (isSymLink(current)) {
+        if (followSymLinks) {
+          return this.findFromPath(resolve(current.path, ...segments), {followSymLinks});
+        } else {
+          break;
+        }
+      }
+    }
+    return {path, entity: current};
+  }
+
+  protected splitIntoFolderAndFile(path: AbsoluteFsPath): [AbsoluteFsPath, string] {
+    const segments = this.splitPath(path);
+    const file = segments.pop() !;
+    return [path.substring(0, path.length - file.length - 1) as AbsoluteFsPath, file];
+  }
+
+  protected ensureFolders(current: Folder, segments: string[]): Folder {
+    // Convert the root folder to a canonical empty string `""` (on Windows it would be `C:`).
+    segments[0] = '';
+    for (const segment of segments) {
+      if (isFile(current[segment])) {
+        throw new Error(`Folder already exists as a file.`);
+      }
+      if (!current[segment]) {
+        current[segment] = {};
+      }
+      current = current[segment] as Folder;
+    }
+    return current;
+  }
+}
+export interface FindResult {
+  path: AbsoluteFsPath;
+  entity: Entity|null;
+}
+export type Entity = Folder | File | SymLink;
+export interface Folder { [pathSegments: string]: Entity; }
+export type File = string;
+export class SymLink {
+  constructor(public path: AbsoluteFsPath) {}
+}
+
+class MockFileStats implements FileStats {
+  constructor(private entity: Entity) {}
+  isFile(): boolean { return isFile(this.entity); }
+  isDirectory(): boolean { return isFolder(this.entity); }
+  isSymbolicLink(): boolean { return isSymLink(this.entity); }
+}
+
+class MockFileSystemError extends Error {
+  constructor(public code: string, public path: string, message: string) { super(message); }
+}
+
+export function isFile(item: Entity | null): item is File {
+  return typeof item === 'string';
+}
+
+export function isSymLink(item: Entity | null): item is SymLink {
+  return item instanceof SymLink;
+}
+
+export function isFolder(item: Entity | null): item is Folder {
+  return item !== null && !isFile(item) && !isSymLink(item);
+}
+
+function cloneFolder(folder: Folder): Folder {
+  const clone: Folder = {};
+  for (const path in folder) {
+    const item = folder[path];
+    if (isSymLink(item)) {
+      clone[path] = new SymLink(item.path);
+    } else if (isFolder(item)) {
+      clone[path] = cloneFolder(item);
+    } else {
+      clone[path] = folder[path];
+    }
+  }
+  return clone;
+}

packages/compiler-cli/src/ngtsc/file_system/testing/src/mock_file_system_native.ts

@@ -0,0 +1,48 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {NodeJSFileSystem} from '../../src/node_js_file_system';
+import {AbsoluteFsPath, PathSegment, PathString} from '../../src/types';
+
+import {MockFileSystem} from './mock_file_system';
+
+export class MockFileSystemNative extends MockFileSystem {
+  constructor(cwd: AbsoluteFsPath = '/' as AbsoluteFsPath) { super(undefined, cwd); }
+
+  // Delegate to the real NodeJSFileSystem for these path related methods
+
+  resolve(...paths: string[]): AbsoluteFsPath {
+    return NodeJSFileSystem.prototype.resolve.call(this, this.pwd(), ...paths);
+  }
+  dirname<T extends string>(file: T): T {
+    return NodeJSFileSystem.prototype.dirname.call(this, file) as T;
+  }
+  join<T extends string>(basePath: T, ...paths: string[]): T {
+    return NodeJSFileSystem.prototype.join.call(this, basePath, ...paths) as T;
+  }
+  relative<T extends PathString>(from: T, to: T): PathSegment {
+    return NodeJSFileSystem.prototype.relative.call(this, from, to);
+  }
+
+  basename(filePath: string, extension?: string): PathSegment {
+    return NodeJSFileSystem.prototype.basename.call(this, filePath, extension);
+  }
+
+  isCaseSensitive() { return NodeJSFileSystem.prototype.isCaseSensitive.call(this); }
+
+  isRooted(path: string): boolean { return NodeJSFileSystem.prototype.isRooted.call(this, path); }
+
+  isRoot(path: AbsoluteFsPath): boolean {
+    return NodeJSFileSystem.prototype.isRoot.call(this, path);
+  }
+
+  normalize<T extends PathString>(path: T): T {
+    return NodeJSFileSystem.prototype.normalize.call(this, path) as T;
+  }
+
+  protected splitPath<T>(path: string): string[] { return path.split(/[\\\/]/); }
+}

packages/compiler-cli/src/ngtsc/file_system/testing/src/mock_file_system_posix.ts

@@ -0,0 +1,41 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+/// <reference types="node" />
+import * as p from 'path';
+
+import {AbsoluteFsPath, PathSegment, PathString} from '../../src/types';
+import {MockFileSystem} from './mock_file_system';
+
+export class MockFileSystemPosix extends MockFileSystem {
+  resolve(...paths: string[]): AbsoluteFsPath {
+    const resolved = p.posix.resolve(this.pwd(), ...paths);
+    return this.normalize(resolved) as AbsoluteFsPath;
+  }
+
+  dirname<T extends string>(file: T): T { return this.normalize(p.posix.dirname(file)) as T; }
+
+  join<T extends string>(basePath: T, ...paths: string[]): T {
+    return this.normalize(p.posix.join(basePath, ...paths)) as T;
+  }
+
+  relative<T extends PathString>(from: T, to: T): PathSegment {
+    return this.normalize(p.posix.relative(from, to)) as PathSegment;
+  }
+
+  basename(filePath: string, extension?: string): PathSegment {
+    return p.posix.basename(filePath, extension) as PathSegment;
+  }
+
+  isRooted(path: string): boolean { return path.startsWith('/'); }
+
+  protected splitPath<T extends PathString>(path: T): string[] { return path.split('/'); }
+
+  normalize<T extends PathString>(path: T): T {
+    return path.replace(/^[a-z]:\//i, '/').replace(/\\/g, '/') as T;
+  }
+}

packages/compiler-cli/src/ngtsc/file_system/testing/src/mock_file_system_windows.ts

@@ -0,0 +1,41 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+/// <reference types="node" />
+import * as p from 'path';
+
+import {AbsoluteFsPath, PathSegment, PathString} from '../../src/types';
+import {MockFileSystem} from './mock_file_system';
+
+export class MockFileSystemWindows extends MockFileSystem {
+  resolve(...paths: string[]): AbsoluteFsPath {
+    const resolved = p.win32.resolve(this.pwd(), ...paths);
+    return this.normalize(resolved as AbsoluteFsPath);
+  }
+
+  dirname<T extends string>(path: T): T { return this.normalize(p.win32.dirname(path) as T); }
+
+  join<T extends string>(basePath: T, ...paths: string[]): T {
+    return this.normalize(p.win32.join(basePath, ...paths)) as T;
+  }
+
+  relative<T extends PathString>(from: T, to: T): PathSegment {
+    return this.normalize(p.win32.relative(from, to)) as PathSegment;
+  }
+
+  basename(filePath: string, extension?: string): PathSegment {
+    return p.win32.basename(filePath, extension) as PathSegment;
+  }
+
+  isRooted(path: string): boolean { return /^([A-Z]:)?([\\\/]|$)/i.test(path); }
+
+  protected splitPath<T extends PathString>(path: T): string[] { return path.split(/[\\\/]/); }
+
+  normalize<T extends PathString>(path: T): T {
+    return path.replace(/^[\/\\]/i, 'C:/').replace(/\\/g, '/') as T;
+  }
+}

packages/compiler-cli/src/ngtsc/file_system/testing/src/test_helper.ts

@@ -0,0 +1,149 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+/// <reference types="jasmine"/>
+import * as ts from 'typescript';
+
+import {absoluteFrom, setFileSystem} from '../../src/helpers';
+import {AbsoluteFsPath} from '../../src/types';
+
+import {MockFileSystem} from './mock_file_system';
+import {MockFileSystemNative} from './mock_file_system_native';
+import {MockFileSystemPosix} from './mock_file_system_posix';
+import {MockFileSystemWindows} from './mock_file_system_windows';
+
+export interface TestFile {
+  name: AbsoluteFsPath;
+  contents: string;
+  isRoot?: boolean|undefined;
+}
+
+export interface RunInEachFileSystemFn {
+  (callback: (os: string) => void): void;
+  windows(callback: (os: string) => void): void;
+  unix(callback: (os: string) => void): void;
+  native(callback: (os: string) => void): void;
+  osX(callback: (os: string) => void): void;
+}
+
+const FS_NATIVE = 'Native';
+const FS_OS_X = 'OS/X';
+const FS_UNIX = 'Unix';
+const FS_WINDOWS = 'Windows';
+const FS_ALL = [FS_OS_X, FS_WINDOWS, FS_UNIX, FS_NATIVE];
+
+function runInEachFileSystemFn(callback: (os: string) => void) {
+  FS_ALL.forEach(os => runInFileSystem(os, callback, false));
+}
+
+function runInFileSystem(os: string, callback: (os: string) => void, error: boolean) {
+  describe(`<<FileSystem: ${os}>>`, () => {
+    beforeEach(() => initMockFileSystem(os));
+    callback(os);
+    if (error) {
+      afterAll(() => { throw new Error(`runInFileSystem limited to ${os}, cannot pass`); });
+    }
+  });
+}
+
+export const runInEachFileSystem: RunInEachFileSystemFn =
+    runInEachFileSystemFn as RunInEachFileSystemFn;
+
+runInEachFileSystem.native = (callback: (os: string) => void) =>
+    runInFileSystem(FS_NATIVE, callback, true);
+runInEachFileSystem.osX = (callback: (os: string) => void) =>
+    runInFileSystem(FS_OS_X, callback, true);
+runInEachFileSystem.unix = (callback: (os: string) => void) =>
+    runInFileSystem(FS_UNIX, callback, true);
+runInEachFileSystem.windows = (callback: (os: string) => void) =>
+    runInFileSystem(FS_WINDOWS, callback, true);
+
+export function initMockFileSystem(os: string, cwd?: AbsoluteFsPath): void {
+  const fs = createMockFileSystem(os, cwd);
+  setFileSystem(fs);
+  monkeyPatchTypeScript(os, fs);
+}
+
+function createMockFileSystem(os: string, cwd?: AbsoluteFsPath): MockFileSystem {
+  switch (os) {
+    case 'OS/X':
+      return new MockFileSystemPosix(/* isCaseSensitive */ false, cwd);
+    case 'Unix':
+      return new MockFileSystemPosix(/* isCaseSensitive */ true, cwd);
+    case 'Windows':
+      return new MockFileSystemWindows(/* isCaseSensitive*/ false, cwd);
+    case 'Native':
+      return new MockFileSystemNative(cwd);
+    default:
+      throw new Error('FileSystem not supported');
+  }
+}
+
+function monkeyPatchTypeScript(os: string, fs: MockFileSystem) {
+  ts.sys.directoryExists = path => {
+    const absPath = fs.resolve(path);
+    return fs.exists(absPath) && fs.stat(absPath).isDirectory();
+  };
+  ts.sys.fileExists = path => {
+    const absPath = fs.resolve(path);
+    return fs.exists(absPath) && fs.stat(absPath).isFile();
+  };
+  ts.sys.getCurrentDirectory = () => fs.pwd();
+  ts.sys.getDirectories = getDirectories;
+  ts.sys.readFile = fs.readFile.bind(fs);
+  ts.sys.resolvePath = fs.resolve.bind(fs);
+  ts.sys.writeFile = fs.writeFile.bind(fs);
+  ts.sys.readDirectory = readDirectory;
+
+  function getDirectories(path: string): string[] {
+    return fs.readdir(absoluteFrom(path)).filter(p => fs.stat(fs.resolve(path, p)).isDirectory());
+  }
+
+  function getFileSystemEntries(path: string): FileSystemEntries {
+    const files: string[] = [];
+    const directories: string[] = [];
+    const absPath = fs.resolve(path);
+    const entries = fs.readdir(absPath);
+    for (const entry of entries) {
+      if (entry == '.' || entry === '..') {
+        continue;
+      }
+      const absPath = fs.resolve(path, entry);
+      const stat = fs.stat(absPath);
+      if (stat.isDirectory()) {
+        directories.push(absPath);
+      } else if (stat.isFile()) {
+        files.push(absPath);
+      }
+    }
+    return {files, directories};
+  }
+
+  function realPath(path: string): string { return fs.realpath(fs.resolve(path)); }
+
+  // Rather than completely re-implementing we are using the `ts.matchFiles` function,
+  // which is internal to the `ts` namespace.
+  const tsMatchFiles: (
+      path: string, extensions: ReadonlyArray<string>| undefined,
+      excludes: ReadonlyArray<string>| undefined, includes: ReadonlyArray<string>| undefined,
+      useCaseSensitiveFileNames: boolean, currentDirectory: string, depth: number | undefined,
+      getFileSystemEntries: (path: string) => FileSystemEntries,
+      realpath: (path: string) => string) => string[] = (ts as any).matchFiles;
+
+  function readDirectory(
+      path: string, extensions?: ReadonlyArray<string>, excludes?: ReadonlyArray<string>,
+      includes?: ReadonlyArray<string>, depth?: number): string[] {
+    return tsMatchFiles(
+        path, extensions, excludes, includes, fs.isCaseSensitive(), fs.pwd(), depth,
+        getFileSystemEntries, realPath);
+  }
+}
+
+interface FileSystemEntries {
+  readonly files: ReadonlyArray<string>;
+  readonly directories: ReadonlyArray<string>;
+}