perf(ivy): ngcc - only find dependencies when targeting a single entry-point (#30525)

Previously, ngcc had to walk the entire `node_modules` tree looking for
entry-points, even if it only needed to process a single target entry-point
and its dependencies.

This added up to a few seconds to each execution of ngcc, which is noticeable
when being run via the CLI integration.

Now, if an entry-point target is provided, only that target and its entry-points
are considered rather than the whole folder tree.

PR Close #30525

packages/compiler-cli/ngcc/src/dependencies/dependency_resolver.ts

@@ -10,7 +10,7 @@ import {DepGraph} from 'dependency-graph';
 import {AbsoluteFsPath, FileSystem, resolve} from '../../../src/ngtsc/file_system';
 import {Logger} from '../logging/logger';
 import {EntryPoint, EntryPointFormat, EntryPointJsonProperty, getEntryPointFormat} from '../packages/entry_point';
-import {DependencyHost} from './dependency_host';
+import {DependencyHost, DependencyInfo} from './dependency_host';
 
 /**
  * Holds information about entry points that are removed because
@@ -98,6 +98,16 @@ export class DependencyResolver {
     };
   }
 
+  getEntryPointDependencies(entryPoint: EntryPoint): DependencyInfo {
+    const formatInfo = this.getEntryPointFormatInfo(entryPoint);
+    const host = this.hosts[formatInfo.format];
+    if (!host) {
+      throw new Error(
+          `Could not find a suitable format for computing dependencies of entry-point: '${entryPoint.path}'.`);
+    }
+    return host.findDependencies(formatInfo.path);
+  }
+
   /**
    * Computes a dependency graph of the given entry-points.
    *
@@ -116,13 +126,7 @@ export class DependencyResolver {
 
     // Now add the dependencies between them
     angularEntryPoints.forEach(entryPoint => {
-      const formatInfo = this.getEntryPointFormatInfo(entryPoint);
-      const host = this.hosts[formatInfo.format];
-      if (!host) {
-        throw new Error(
-            `Could not find a suitable format for computing dependencies of entry-point: '${entryPoint.path}'.`);
-      }
-      const {dependencies, missing, deepImports} = host.findDependencies(formatInfo.path);
+      const {dependencies, missing, deepImports} = this.getEntryPointDependencies(entryPoint);
 
       if (missing.size > 0) {
         // This entry point has dependencies that are missing

packages/compiler-cli/ngcc/src/entry_point_finder/directory_walker_entry_point_finder.ts

@@ -8,61 +8,31 @@
 import {AbsoluteFsPath, FileSystem, join, resolve} from '../../../src/ngtsc/file_system';
 import {DependencyResolver, SortedEntryPointsInfo} from '../dependencies/dependency_resolver';
 import {Logger} from '../logging/logger';
+import {NgccConfiguration} from '../packages/configuration';
+import {EntryPoint, getEntryPointInfo} from '../packages/entry_point';
 import {PathMappings} from '../utils';
-import {NgccConfiguration} from './configuration';
-import {EntryPoint, getEntryPointInfo} from './entry_point';
+import {EntryPointFinder} from './interface';
+import {getBasePaths} from './utils';
 
-export class EntryPointFinder {
+/**
+ * An EntryPointFinder that searches for all entry-points that can be found given a `basePath` and
+ * `pathMappings`.
+ */
+export class DirectoryWalkerEntryPointFinder implements EntryPointFinder {
+  private basePaths = getBasePaths(this.sourceDirectory, this.pathMappings);
   constructor(
       private fs: FileSystem, private config: NgccConfiguration, private logger: Logger,
-      private resolver: DependencyResolver) {}
+      private resolver: DependencyResolver, private sourceDirectory: AbsoluteFsPath,
+      private pathMappings: PathMappings|undefined) {}
   /**
-   * Search the given directory, and sub-directories, for Angular package entry points.
-   * @param sourceDirectory An absolute path to the directory to search for entry points.
+   * Search the `sourceDirectory`, and sub-directories, using `pathMappings` as necessary, to find
+   * all package entry-points.
    */
-  findEntryPoints(
-      sourceDirectory: AbsoluteFsPath, targetEntryPointPath?: AbsoluteFsPath,
-      pathMappings?: PathMappings): SortedEntryPointsInfo {
-    const basePaths = this.getBasePaths(sourceDirectory, pathMappings);
-    const unsortedEntryPoints = basePaths.reduce<EntryPoint[]>(
+  findEntryPoints(): SortedEntryPointsInfo {
+    const unsortedEntryPoints = this.basePaths.reduce<EntryPoint[]>(
         (entryPoints, basePath) => entryPoints.concat(this.walkDirectoryForEntryPoints(basePath)),
         []);
-    const targetEntryPoint = targetEntryPointPath ?
-        unsortedEntryPoints.find(entryPoint => entryPoint.path === targetEntryPointPath) :
-        undefined;
-    return this.resolver.sortEntryPointsByDependency(unsortedEntryPoints, targetEntryPoint);
-  }
-
-  /**
-   * Extract all the base-paths that we need to search for entry-points.
-   *
-   * This always contains the standard base-path (`sourceDirectory`).
-   * But it also parses the `paths` mappings object to guess additional base-paths.
-   *
-   * For example:
-   *
-   * ```
-   * getBasePaths('/node_modules', {baseUrl: '/dist', paths: {'*': ['lib/*', 'lib/generated/*']}})
-   * > ['/node_modules', '/dist/lib']
-   * ```
-   *
-   * Notice that `'/dist'` is not included as there is no `'*'` path,
-   * and `'/dist/lib/generated'` is not included as it is covered by `'/dist/lib'`.
-   *
-   * @param sourceDirectory The standard base-path (e.g. node_modules).
-   * @param pathMappings Path mapping configuration, from which to extract additional base-paths.
-   */
-  private getBasePaths(sourceDirectory: AbsoluteFsPath, pathMappings?: PathMappings):
-      AbsoluteFsPath[] {
-    const basePaths = [sourceDirectory];
-    if (pathMappings) {
-      const baseUrl = resolve(pathMappings.baseUrl);
-      values(pathMappings.paths).forEach(paths => paths.forEach(path => {
-        basePaths.push(join(baseUrl, extractPathPrefix(path)));
-      }));
-    }
-    basePaths.sort();  // Get the paths in order with the shorter ones first.
-    return basePaths.filter(removeDeeperPaths);
+    return this.resolver.sortEntryPointsByDependency(unsortedEntryPoints);
   }
 
   /**
@@ -168,38 +138,6 @@ export class EntryPointFinder {
   }
 }
 
-/**
- * Extract everything in the `path` up to the first `*`.
- * @param path The path to parse.
- * @returns The extracted prefix.
- */
-function extractPathPrefix(path: string) {
-  return path.split('*', 1)[0];
-}
-
-/**
- * A filter function that removes paths that are already covered by higher paths.
- *
- * @param value The current path.
- * @param index The index of the current path.
- * @param array The array of paths (sorted alphabetically).
- * @returns true if this path is not already covered by a previous path.
- */
-function removeDeeperPaths(value: AbsoluteFsPath, index: number, array: AbsoluteFsPath[]) {
-  for (let i = 0; i < index; i++) {
-    if (value.startsWith(array[i])) return false;
-  }
-  return true;
-}
-
-/**
- * Extract all the values (not keys) from an object.
- * @param obj The object to process.
- */
-function values<T>(obj: {[key: string]: T}): T[] {
-  return Object.keys(obj).map(key => obj[key]);
-}
-
 function stripJsExtension<T extends string>(filePath: T): T {
   return filePath.replace(/\.js$/, '') as T;
 }

packages/compiler-cli/ngcc/src/entry_point_finder/interface.ts

@@ -0,0 +1,15 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {SortedEntryPointsInfo} from '../dependencies/dependency_resolver';
+
+export interface EntryPointFinder {
+  /**
+   * Search for Angular package entry-points.
+   */
+  findEntryPoints(): SortedEntryPointsInfo;
+}

packages/compiler-cli/ngcc/src/entry_point_finder/targeted_entry_point_finder.ts

@@ -0,0 +1,123 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {AbsoluteFsPath, FileSystem, PathSegment, join, relative, relativeFrom} from '../../../src/ngtsc/file_system';
+import {DependencyResolver, SortedEntryPointsInfo} from '../dependencies/dependency_resolver';
+import {Logger} from '../logging/logger';
+import {NgccConfiguration} from '../packages/configuration';
+import {EntryPoint, getEntryPointInfo} from '../packages/entry_point';
+import {PathMappings} from '../utils';
+import {EntryPointFinder} from './interface';
+import {getBasePaths} from './utils';
+
+/**
+ * An EntryPointFinder that starts from a target entry-point and only finds
+ * entry-points that are dependencies of the target.
+ *
+ * This is faster than searching the entire file-system for all the entry-points,
+ * and is used primarily by the CLI integration.
+ */
+export class TargetedEntryPointFinder implements EntryPointFinder {
+  private unprocessedPaths: AbsoluteFsPath[] = [];
+  private unsortedEntryPoints = new Map<AbsoluteFsPath, EntryPoint>();
+  private basePaths = getBasePaths(this.basePath, this.pathMappings);
+
+  constructor(
+      private fs: FileSystem, private config: NgccConfiguration, private logger: Logger,
+      private resolver: DependencyResolver, private basePath: AbsoluteFsPath,
+      private targetPath: AbsoluteFsPath, private pathMappings: PathMappings|undefined) {}
+
+  findEntryPoints(): SortedEntryPointsInfo {
+    this.unprocessedPaths = [this.targetPath];
+    while (this.unprocessedPaths.length > 0) {
+      this.processNextPath();
+    }
+    const targetEntryPoint = this.unsortedEntryPoints.get(this.targetPath);
+    return this.resolver.sortEntryPointsByDependency(
+        Array.from(this.unsortedEntryPoints.values()), targetEntryPoint);
+  }
+
+  private processNextPath(): void {
+    const path = this.unprocessedPaths.shift() !;
+    const entryPoint = this.getEntryPoint(path);
+    if (entryPoint !== null) {
+      this.unsortedEntryPoints.set(entryPoint.path, entryPoint);
+      const deps = this.resolver.getEntryPointDependencies(entryPoint);
+      deps.dependencies.forEach(dep => {
+        if (!this.unsortedEntryPoints.has(dep)) {
+          this.unprocessedPaths.push(dep);
+        }
+      });
+    }
+  }
+
+  private getEntryPoint(entryPointPath: AbsoluteFsPath): EntryPoint|null {
+    const packagePath = this.computePackagePath(entryPointPath);
+    return getEntryPointInfo(this.fs, this.config, this.logger, packagePath, entryPointPath);
+  }
+
+  /**
+   * Search down to the `entryPointPath` from each `basePath` for the first `package.json` that we
+   * come to. This is the path to the entry-point's containing package. For example if `basePath` is
+   * `/a/b/c` and `entryPointPath` is `/a/b/c/d/e` and there exists `/a/b/c/d/package.json` and
+   * `/a/b/c/d/e/package.json`, then we will return `/a/b/c/d`.
+   *
+   * To account for nested `node_modules` we actually start the search at the last `node_modules` in
+   * the `entryPointPath` that is below the `basePath`. E.g. if `basePath` is `/a/b/c` and
+   * `entryPointPath` is `/a/b/c/d/node_modules/x/y/z`, we start the search at
+   * `/a/b/c/d/node_modules`.
+   */
+  private computePackagePath(entryPointPath: AbsoluteFsPath): AbsoluteFsPath {
+    for (const basePath of this.basePaths) {
+      if (entryPointPath.startsWith(basePath)) {
+        let packagePath = basePath;
+        const segments = this.splitPath(relative(basePath, entryPointPath));
+
+        // Start the search at the last nested `node_modules` folder if the relative
+        // `entryPointPath` contains one or more.
+        let nodeModulesIndex = segments.lastIndexOf(relativeFrom('node_modules'));
+        while (nodeModulesIndex >= 0) {
+          packagePath = join(packagePath, segments.shift() !);
+          nodeModulesIndex--;
+        }
+
+        // Note that we skip the first `packagePath` and start looking from the first folder below
+        // it because that will be the `node_modules` folder.
+        for (const segment of segments) {
+          packagePath = join(packagePath, segment);
+          if (this.fs.exists(join(packagePath, 'package.json'))) {
+            return packagePath;
+          }
+        }
+
+        // If we got here then we couldn't find a `packagePath` for the current `basePath` but since
+        // `basePath`s are guaranteed not to be a sub-directory each other then no other `basePath`
+        // will match either.
+        break;
+      }
+    }
+    // If we get here then none of the `basePaths` matched the `entryPointPath`, which
+    // is somewhat unexpected and means that this entry-point lives completely outside
+    // any of the `basePaths`.
+    // All we can do is assume that his entry-point is a primary entry-point to a package.
+    return entryPointPath;
+  }
+
+
+  /**
+   * Split the given `path` into path segments using an FS independent algorithm.
+   * @param path The path to split.
+   */
+  private splitPath(path: PathSegment) {
+    const segments = [];
+    while (path !== '.') {
+      segments.unshift(this.fs.basename(path));
+      path = this.fs.dirname(path);
+    }
+    return segments;
+  }
+}

packages/compiler-cli/ngcc/src/entry_point_finder/utils.ts

@@ -0,0 +1,65 @@
+/**
+ * @license
+ * Copyright Google Inc. All Rights Reserved.
+ *
+ * Use of this source code is governed by an MIT-style license that can be
+ * found in the LICENSE file at https://angular.io/license
+ */
+import {AbsoluteFsPath, join, resolve} from '../../../src/ngtsc/file_system';
+import {PathMappings} from '../utils';
+
+/**
+ * Extract all the base-paths that we need to search for entry-points.
+ *
+ * This always contains the standard base-path (`sourceDirectory`).
+ * But it also parses the `paths` mappings object to guess additional base-paths.
+ *
+ * For example:
+ *
+ * ```
+ * getBasePaths('/node_modules', {baseUrl: '/dist', paths: {'*': ['lib/*', 'lib/generated/*']}})
+ * > ['/node_modules', '/dist/lib']
+ * ```
+ *
+ * Notice that `'/dist'` is not included as there is no `'*'` path,
+ * and `'/dist/lib/generated'` is not included as it is covered by `'/dist/lib'`.
+ *
+ * @param sourceDirectory The standard base-path (e.g. node_modules).
+ * @param pathMappings Path mapping configuration, from which to extract additional base-paths.
+ */
+export function getBasePaths(
+    sourceDirectory: AbsoluteFsPath, pathMappings: PathMappings | undefined): AbsoluteFsPath[] {
+  const basePaths = [sourceDirectory];
+  if (pathMappings) {
+    const baseUrl = resolve(pathMappings.baseUrl);
+    Object.values(pathMappings.paths).forEach(paths => paths.forEach(path => {
+      basePaths.push(join(baseUrl, extractPathPrefix(path)));
+    }));
+  }
+  basePaths.sort();  // Get the paths in order with the shorter ones first.
+  return basePaths.filter(removeDeeperPaths);
+}
+
+/**
+ * Extract everything in the `path` up to the first `*`.
+ * @param path The path to parse.
+ * @returns The extracted prefix.
+ */
+function extractPathPrefix(path: string) {
+  return path.split('*', 1)[0];
+}
+
+/**
+ * A filter function that removes paths that are already covered by higher paths.
+ *
+ * @param value The current path.
+ * @param index The index of the current path.
+ * @param array The array of paths (sorted alphabetically).
+ * @returns true if this path is not already covered by a previous path.
+ */
+function removeDeeperPaths(value: AbsoluteFsPath, index: number, array: AbsoluteFsPath[]) {
+  for (let i = 0; i < index; i++) {
+    if (value.startsWith(array[i])) return false;
+  }
+  return true;
+}

packages/compiler-cli/ngcc/src/main.ts

@@ -7,17 +7,18 @@
  */
 import {AbsoluteFsPath, FileSystem, absoluteFrom, dirname, getFileSystem, resolve} from '../../src/ngtsc/file_system';
 import {CommonJsDependencyHost} from './dependencies/commonjs_dependency_host';
-import {DependencyResolver} from './dependencies/dependency_resolver';
+import {DependencyResolver, InvalidEntryPoint, SortedEntryPointsInfo} from './dependencies/dependency_resolver';
 import {EsmDependencyHost} from './dependencies/esm_dependency_host';
 import {ModuleResolver} from './dependencies/module_resolver';
 import {UmdDependencyHost} from './dependencies/umd_dependency_host';
+import {DirectoryWalkerEntryPointFinder} from './entry_point_finder/directory_walker_entry_point_finder';
+import {TargetedEntryPointFinder} from './entry_point_finder/targeted_entry_point_finder';
 import {ConsoleLogger, LogLevel} from './logging/console_logger';
 import {Logger} from './logging/logger';
 import {hasBeenProcessed, markAsProcessed} from './packages/build_marker';
 import {NgccConfiguration} from './packages/configuration';
-import {EntryPointFormat, EntryPointJsonProperty, SUPPORTED_FORMAT_PROPERTIES, getEntryPointFormat} from './packages/entry_point';
+import {EntryPoint, EntryPointFormat, EntryPointJsonProperty, SUPPORTED_FORMAT_PROPERTIES, getEntryPointFormat} from './packages/entry_point';
 import {makeEntryPointBundle} from './packages/entry_point_bundle';
-import {EntryPointFinder} from './packages/entry_point_finder';
 import {Transformer} from './packages/transformer';
 import {PathMappings} from './utils';
 import {FileWriter} from './writing/file_writer';
@@ -93,36 +94,12 @@ export function mainNgcc(
     umd: umdDependencyHost,
     commonjs: commonJsDependencyHost
   });
-  const config = new NgccConfiguration(fileSystem, dirname(absoluteFrom(basePath)));
-  const finder = new EntryPointFinder(fileSystem, config, logger, resolver);
+  const absBasePath = absoluteFrom(basePath);
+  const config = new NgccConfiguration(fileSystem, dirname(absBasePath));
   const fileWriter = getFileWriter(fileSystem, createNewEntryPointFormats);
-
-  const absoluteTargetEntryPointPath =
-      targetEntryPointPath ? resolve(basePath, targetEntryPointPath) : undefined;
-
-  if (absoluteTargetEntryPointPath &&
-      hasProcessedTargetEntryPoint(
-          fileSystem, absoluteTargetEntryPointPath, propertiesToConsider, compileAllFormats)) {
-    logger.debug('The target entry-point has already been processed');
-    return;
-  }
-
-  const {entryPoints, invalidEntryPoints} =
-      finder.findEntryPoints(absoluteFrom(basePath), absoluteTargetEntryPointPath, pathMappings);
-
-  invalidEntryPoints.forEach(invalidEntryPoint => {
-    logger.debug(
-        `Invalid entry-point ${invalidEntryPoint.entryPoint.path}.`,
-        `It is missing required dependencies:\n` +
-            invalidEntryPoint.missingDependencies.map(dep => ` - ${dep}`).join('\n'));
-  });
-
-  if (absoluteTargetEntryPointPath && entryPoints.length === 0) {
-    markNonAngularPackageAsProcessed(
-        fileSystem, absoluteTargetEntryPointPath, propertiesToConsider);
-    return;
-  }
-
+  const entryPoints = getEntryPoints(
+      fileSystem, config, logger, resolver, absBasePath, targetEntryPointPath, pathMappings,
+      propertiesToConsider, compileAllFormats);
   for (const entryPoint of entryPoints) {
     // Are we compiling the Angular core?
     const isCore = entryPoint.name === '@angular/core';
@@ -188,6 +165,47 @@ function getFileWriter(fs: FileSystem, createNewEntryPointFormats: boolean): Fil
   return createNewEntryPointFormats ? new NewEntryPointFileWriter(fs) : new InPlaceFileWriter(fs);
 }
 
+function getEntryPoints(
+    fs: FileSystem, config: NgccConfiguration, logger: Logger, resolver: DependencyResolver,
+    basePath: AbsoluteFsPath, targetEntryPointPath: string | undefined,
+    pathMappings: PathMappings | undefined, propertiesToConsider: string[],
+    compileAllFormats: boolean): EntryPoint[] {
+  const {entryPoints, invalidEntryPoints} = (targetEntryPointPath !== undefined) ?
+      getTargetedEntryPoints(
+          fs, config, logger, resolver, basePath, targetEntryPointPath, propertiesToConsider,
+          compileAllFormats, pathMappings) :
+      getAllEntryPoints(fs, config, logger, resolver, basePath, pathMappings);
+  logInvalidEntryPoints(logger, invalidEntryPoints);
+  return entryPoints;
+}
+
+function getTargetedEntryPoints(
+    fs: FileSystem, config: NgccConfiguration, logger: Logger, resolver: DependencyResolver,
+    basePath: AbsoluteFsPath, targetEntryPointPath: string, propertiesToConsider: string[],
+    compileAllFormats: boolean, pathMappings: PathMappings | undefined): SortedEntryPointsInfo {
+  const absoluteTargetEntryPointPath = resolve(basePath, targetEntryPointPath);
+  if (hasProcessedTargetEntryPoint(
+          fs, absoluteTargetEntryPointPath, propertiesToConsider, compileAllFormats)) {
+    logger.debug('The target entry-point has already been processed');
+    return {entryPoints: [], invalidEntryPoints: [], ignoredDependencies: []};
+  }
+  const finder = new TargetedEntryPointFinder(
+      fs, config, logger, resolver, basePath, absoluteTargetEntryPointPath, pathMappings);
+  const entryPointInfo = finder.findEntryPoints();
+  if (entryPointInfo.entryPoints.length === 0) {
+    markNonAngularPackageAsProcessed(fs, absoluteTargetEntryPointPath, propertiesToConsider);
+  }
+  return entryPointInfo;
+}
+
+function getAllEntryPoints(
+    fs: FileSystem, config: NgccConfiguration, logger: Logger, resolver: DependencyResolver,
+    basePath: AbsoluteFsPath, pathMappings: PathMappings | undefined): SortedEntryPointsInfo {
+  const finder =
+      new DirectoryWalkerEntryPointFinder(fs, config, logger, resolver, basePath, pathMappings);
+  return finder.findEntryPoints();
+}
+
 function hasProcessedTargetEntryPoint(
     fs: FileSystem, targetPath: AbsoluteFsPath, propertiesToConsider: string[],
     compileAllFormats: boolean) {
@@ -233,3 +251,12 @@ function markNonAngularPackageAsProcessed(
       markAsProcessed(fs, packageJson, packageJsonPath, formatProperty as EntryPointJsonProperty);
   });
 }
+
+function logInvalidEntryPoints(logger: Logger, invalidEntryPoints: InvalidEntryPoint[]): void {
+  invalidEntryPoints.forEach(invalidEntryPoint => {
+    logger.debug(
+        `Invalid entry-point ${invalidEntryPoint.entryPoint.path}.`,
+        `It is missing required dependencies:\n` +
+            invalidEntryPoint.missingDependencies.map(dep => ` - ${dep}`).join('\n'));
+  });
+}