feat(ngcc): automatically clean outdated ngcc artifacts (#35079)

If ngcc gets updated to a new version then the artifacts
left in packages that were processed by the previous
version are possibly invalid.

Previously we just errored if we found packages that
had already been processed by an outdated version.

Now we automatically clean the packages that have
outdated artifacts so that they can be reprocessed
correctly with the current ngcc version.

Fixes #35082

PR Close #35079

packages/compiler-cli/ngcc/src/main.ts

@@ -38,12 +38,12 @@ import {EntryPoint, EntryPointJsonProperty, EntryPointPackageJson, SUPPORTED_FOR
 import {makeEntryPointBundle} from './packages/entry_point_bundle';
 import {Transformer} from './packages/transformer';
 import {PathMappings} from './utils';
+import {cleanOutdatedPackages} from './writing/cleaning/package_cleaner';
 import {FileWriter} from './writing/file_writer';
 import {InPlaceFileWriter} from './writing/in_place_file_writer';
 import {NewEntryPointFileWriter} from './writing/new_entry_point_file_writer';
 import {DirectPackageJsonUpdater, PackageJsonUpdater} from './writing/package_json_updater';
 
-
 /**
  * The options to configure the ngcc compiler for synchronous execution.
  */
@@ -188,10 +188,19 @@ export function mainNgcc(
 
     const absBasePath = absoluteFrom(basePath);
     const config = new NgccConfiguration(fileSystem, dirname(absBasePath));
-    const {entryPoints, graph} = getEntryPoints(
+    let entryPointInfo = getEntryPoints(
         fileSystem, pkgJsonUpdater, logger, dependencyResolver, config, absBasePath,
         absoluteTargetEntryPointPath, pathMappings);
 
+    const cleaned = cleanOutdatedPackages(fileSystem, entryPointInfo.entryPoints);
+    if (cleaned) {
+      // If we had to clean up one or more packages then we must read in the entry-points again.
+      entryPointInfo = getEntryPoints(
+          fileSystem, pkgJsonUpdater, logger, dependencyResolver, config, absBasePath,
+          absoluteTargetEntryPointPath, pathMappings);
+    }
+    const {entryPoints, graph} = entryPointInfo;
+
     const unprocessableEntryPointPaths: string[] = [];
     // The tasks are partially ordered by virtue of the entry-points being partially ordered too.
     const tasks: PartiallyOrderedTasks = [] as any;

packages/compiler-cli/ngcc/src/writing/cleaning/cleaning_strategies.ts

@@ -0,0 +1,63 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {AbsoluteFsPath, FileSystem, PathSegment, absoluteFrom} from '../../../../src/ngtsc/file_system';
+import {cleanPackageJson} from '../../packages/build_marker';
+import {NGCC_BACKUP_EXTENSION} from '../in_place_file_writer';
+import {NGCC_DIRECTORY} from '../new_entry_point_file_writer';
+import {isLocalDirectory} from './utils';
+
+/**
+* Implement this interface to extend the cleaning strategies of the `PackageCleaner`.
+*/
+export interface CleaningStrategy {
+  canClean(path: AbsoluteFsPath, basename: PathSegment): boolean;
+  clean(path: AbsoluteFsPath, basename: PathSegment): void;
+}
+
+/**
+ * A CleaningStrategy that reverts changes to package.json files by removing the build marker and
+ * other properties.
+ */
+export class PackageJsonCleaner implements CleaningStrategy {
+  constructor(private fs: FileSystem) {}
+  canClean(_path: AbsoluteFsPath, basename: PathSegment): boolean {
+    return basename === 'package.json';
+  }
+  clean(path: AbsoluteFsPath, _basename: PathSegment): void {
+    const packageJson = JSON.parse(this.fs.readFile(path));
+    if (cleanPackageJson(packageJson)) {
+      this.fs.writeFile(path, `${JSON.stringify(packageJson, null, 2)}\n`);
+    }
+  }
+}
+
+/**
+ * A CleaningStrategy that removes the extra directory containing generated entry-point formats.
+ */
+export class NgccDirectoryCleaner implements CleaningStrategy {
+  constructor(private fs: FileSystem) {}
+  canClean(path: AbsoluteFsPath, basename: PathSegment): boolean {
+    return basename === NGCC_DIRECTORY && isLocalDirectory(this.fs, path);
+  }
+  clean(path: AbsoluteFsPath, _basename: PathSegment): void { this.fs.removeDeep(path); }
+}
+
+/**
+ * A CleaningStrategy that reverts files that were overwritten and removes the backup files that
+ * ngcc created.
+ */
+export class BackupFileCleaner implements CleaningStrategy {
+  constructor(private fs: FileSystem) {}
+  canClean(path: AbsoluteFsPath, basename: PathSegment): boolean {
+    return this.fs.extname(basename) === NGCC_BACKUP_EXTENSION &&
+        this.fs.exists(absoluteFrom(path.replace(NGCC_BACKUP_EXTENSION, '')));
+  }
+  clean(path: AbsoluteFsPath, _basename: PathSegment): void {
+    this.fs.moveFile(path, absoluteFrom(path.replace(NGCC_BACKUP_EXTENSION, '')));
+  }
+}

packages/compiler-cli/ngcc/src/writing/cleaning/package_cleaner.ts

@@ -0,0 +1,80 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {AbsoluteFsPath, FileSystem} from '../../../../src/ngtsc/file_system';
+import {needsCleaning} from '../../packages/build_marker';
+import {EntryPoint} from '../../packages/entry_point';
+
+import {BackupFileCleaner, CleaningStrategy, NgccDirectoryCleaner, PackageJsonCleaner} from './cleaning_strategies';
+import {isLocalDirectory} from './utils';
+
+/**
+ * A class that can clean ngcc artifacts from a directory.
+ */
+export class PackageCleaner {
+  constructor(private fs: FileSystem, private cleaners: CleaningStrategy[]) {}
+
+  /**
+   * Recurse through the file-system cleaning files and directories as determined by the configured
+   * cleaning-strategies.
+   *
+   * @param directory the current directory to clean
+   */
+  clean(directory: AbsoluteFsPath) {
+    const basenames = this.fs.readdir(directory);
+    for (const basename of basenames) {
+      if (basename === 'node_modules') {
+        continue;
+      }
+
+      const path = this.fs.resolve(directory, basename);
+      for (const cleaner of this.cleaners) {
+        if (cleaner.canClean(path, basename)) {
+          cleaner.clean(path, basename);
+          break;
+        }
+      }
+      // Recurse into subdirectories (note that a cleaner may have removed this path)
+      if (isLocalDirectory(this.fs, path)) {
+        this.clean(path);
+      }
+    }
+  }
+}
+
+
+/**
+ * Iterate through the given `entryPoints` identifying the package for each that has at least one
+ * outdated processed format, then cleaning those packages.
+ *
+ * Note that we have to clean entire packages because there is no clear file-system boundary
+ * between entry-points within a package. So if one entry-point is outdated we have to clean
+ * everything within that package.
+ *
+ * @param fileSystem the current file-system
+ * @param entryPoints the entry-points that have been collected for this run of ngcc
+ * @returns true if packages needed to be cleaned.
+ */
+export function cleanOutdatedPackages(fileSystem: FileSystem, entryPoints: EntryPoint[]): boolean {
+  const packagesToClean = new Set<AbsoluteFsPath>();
+  for (const entryPoint of entryPoints) {
+    if (needsCleaning(entryPoint.packageJson)) {
+      packagesToClean.add(entryPoint.package);
+    }
+  }
+
+  const cleaner = new PackageCleaner(fileSystem, [
+    new PackageJsonCleaner(fileSystem),
+    new NgccDirectoryCleaner(fileSystem),
+    new BackupFileCleaner(fileSystem),
+  ]);
+  for (const packagePath of packagesToClean) {
+    cleaner.clean(packagePath);
+  }
+
+  return packagesToClean.size > 0;
+}

packages/compiler-cli/ngcc/src/writing/cleaning/utils.ts

@@ -0,0 +1,23 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {AbsoluteFsPath, FileSystem} from '../../../../src/ngtsc/file_system';
+
+/**
+ * Returns true if the given `path` is a directory (not a symlink) and actually exists.
+ *
+ * @param fs the current filesystem
+ * @param path the path to check
+ */
+export function isLocalDirectory(fs: FileSystem, path: AbsoluteFsPath): boolean {
+  if (fs.exists(path)) {
+    const stat = fs.lstat(path);
+    return stat.isDirectory();
+  } else {
+    return false;
+  }
+}