perf(ivy): reuse prior analysis work during incremental builds (#34288)

Previously, the compiler performed an incremental build by analyzing and
resolving all classes in the program (even unchanged ones) and then using
the dependency graph information to determine which .js files were stale and
needed to be re-emitted. This algorithm produced "correct" rebuilds, but the
cost of re-analyzing the entire program turned out to be higher than
anticipated, especially for component-heavy compilations.

To achieve performant rebuilds, it is necessary to reuse previous analysis
results if possible. Doing this safely requires knowing when prior work is
viable and when it is stale and needs to be re-done.

The new algorithm implemented by this commit is such:

1) Each incremental build starts with knowledge of the last known good
   dependency graph and analysis results from the last successful build,
   plus of course information about the set of files changed.

2) The previous dependency graph's information is used to determine the
   set of source files which have "logically" changed. A source file is
   considered logically changed if it or any of its dependencies have
   physically changed (on disk) since the last successful compilation. Any
   logically unchanged dependencies have their dependency information copied
   over to the new dependency graph.

3) During the `TraitCompiler`'s loop to consider all source files in the
   program, if a source file is logically unchanged then its previous
   analyses are "adopted" (and their 'register' steps are run). If the file
   is logically changed, then it is re-analyzed as usual.

4) Then, incremental build proceeds as before, with the new dependency graph
   being used to determine the set of files which require re-emitting.

This analysis reuse avoids template parsing operations in many circumstances
and significantly reduces the time it takes ngtsc to rebuild a large
application.

Future work will increase performance even more, by tackling a variety of
other opportunities to reuse or avoid work.

PR Close #34288

packages/compiler-cli/src/ngtsc/transform/BUILD.bazel

@@ -11,6 +11,7 @@ ts_library(
         "//packages/compiler",
         "//packages/compiler-cli/src/ngtsc/diagnostics",
         "//packages/compiler-cli/src/ngtsc/imports",
+        "//packages/compiler-cli/src/ngtsc/incremental:api",
         "//packages/compiler-cli/src/ngtsc/indexer",
         "//packages/compiler-cli/src/ngtsc/modulewithproviders",
         "//packages/compiler-cli/src/ngtsc/perf",

packages/compiler-cli/src/ngtsc/transform/index.ts

@@ -7,6 +7,6 @@
  */
 
 export * from './src/api';
-export {TraitCompiler} from './src/compilation';
+export {ClassRecord, TraitCompiler} from './src/compilation';
 export {declarationTransformFactory, DtsTransformRegistry, IvyDeclarationDtsTransform, ReturnTypeTransform} from './src/declaration';
 export {ivyTransformFactory} from './src/transform';

packages/compiler-cli/src/ngtsc/transform/src/api.ts

@@ -73,6 +73,8 @@ export enum HandlerFlags {
  * @param `R` The type of resolution metadata produced by `resolve`.
  */
 export interface DecoratorHandler<D, A, R> {
+  readonly name: string;
+
   /**
    * The precedence of a handler controls how it interacts with other handlers that match the same
    * class.

packages/compiler-cli/src/ngtsc/transform/src/compilation.ts

@@ -11,6 +11,7 @@ import * as ts from 'typescript';
 
 import {ErrorCode, FatalDiagnosticError} from '../../diagnostics';
 import {ImportRewriter} from '../../imports';
+import {IncrementalBuild} from '../../incremental/api';
 import {IndexingContext} from '../../indexer';
 import {ModuleWithProvidersScanner} from '../../modulewithproviders';
 import {PerfRecorder} from '../../perf';
@@ -22,10 +23,11 @@ import {AnalysisOutput, CompileResult, DecoratorHandler, DetectResult, HandlerPr
 import {DtsTransformRegistry} from './declaration';
 import {Trait, TraitState} from './trait';
 
+
 /**
  * Records information about a specific class that has matched traits.
  */
-interface ClassRecord {
+export interface ClassRecord {
   /**
    * The `ClassDeclaration` of the class which has Angular traits applied.
    */
@@ -59,7 +61,13 @@ interface ClassRecord {
 /**
  * The heart of Angular compilation.
  *
- * The `TraitCompiler` is responsible for processing all classes in the program and
+ * The `TraitCompiler` is responsible for processing all classes in the program. Any time a
+ * `DecoratorHandler` matches a class, a "trait" is created to represent that Angular aspect of the
+ * class (such as the class having a component definition).
+ *
+ * The `TraitCompiler` transitions each trait through the various phases of compilation, culminating
+ * in the production of `CompileResult`s instructing the compiler to apply various mutations to the
+ * class (like adding fields or type declarations).
  */
 export class TraitCompiler {
   /**
@@ -76,19 +84,17 @@ export class TraitCompiler {
 
   private reexportMap = new Map<string, Map<string, [string, string]>>();
 
-  /**
-   * @param handlers array of `DecoratorHandler`s which will be executed against each class in the
-   * program
-   * @param checker TypeScript `TypeChecker` instance for the program
-   * @param reflector `ReflectionHost` through which all reflection operations will be performed
-   * @param coreImportsFrom a TypeScript `SourceFile` which exports symbols needed for Ivy imports
-   * when compiling @angular/core, or `null` if the current program is not @angular/core. This is
-   * `null` in most cases.
-   */
+  private handlersByName = new Map<string, DecoratorHandler<unknown, unknown, unknown>>();
+
   constructor(
       private handlers: DecoratorHandler<unknown, unknown, unknown>[],
       private reflector: ReflectionHost, private perf: PerfRecorder,
-      private compileNonExportedClasses: boolean, private dtsTransforms: DtsTransformRegistry) {}
+      private incrementalBuild: IncrementalBuild<ClassRecord>,
+      private compileNonExportedClasses: boolean, private dtsTransforms: DtsTransformRegistry) {
+    for (const handler of handlers) {
+      this.handlersByName.set(handler.name, handler);
+    }
+  }
 
   analyzeSync(sf: ts.SourceFile): void { this.analyze(sf, false); }
 
@@ -101,6 +107,16 @@ export class TraitCompiler {
     // type of 'void', so `undefined` is used instead.
     const promises: Promise<void>[] = [];
 
+    const priorWork = this.incrementalBuild.priorWorkFor(sf);
+    if (priorWork !== null) {
+      for (const priorRecord of priorWork) {
+        this.adopt(priorRecord);
+      }
+
+      // Skip the rest of analysis, as this file's prior traits are being reused.
+      return;
+    }
+
     const visit = (node: ts.Node): void => {
       if (isNamedClassDeclaration(node)) {
         this.analyzeClass(node, preanalyze ? promises : null);
@@ -117,6 +133,60 @@ export class TraitCompiler {
     }
   }
 
+  recordsFor(sf: ts.SourceFile): ClassRecord[]|null {
+    if (!this.fileToClasses.has(sf)) {
+      return null;
+    }
+    const records: ClassRecord[] = [];
+    for (const clazz of this.fileToClasses.get(sf) !) {
+      records.push(this.classes.get(clazz) !);
+    }
+    return records;
+  }
+
+  /**
+   * Import a `ClassRecord` from a previous compilation.
+   *
+   * Traits from the `ClassRecord` have accurate metadata, but the `handler` is from the old program
+   * and needs to be updated (matching is done by name). A new pending trait is created and then
+   * transitioned to analyzed using the previous analysis. If the trait is in the errored state,
+   * instead the errors are copied over.
+   */
+  private adopt(priorRecord: ClassRecord): void {
+    const record: ClassRecord = {
+      hasPrimaryHandler: priorRecord.hasPrimaryHandler,
+      hasWeakHandlers: priorRecord.hasWeakHandlers,
+      metaDiagnostics: priorRecord.metaDiagnostics,
+      node: priorRecord.node,
+      traits: [],
+    };
+
+    for (const priorTrait of priorRecord.traits) {
+      const handler = this.handlersByName.get(priorTrait.handler.name) !;
+      let trait: Trait<unknown, unknown, unknown> = Trait.pending(handler, priorTrait.detected);
+
+      if (priorTrait.state === TraitState.ANALYZED || priorTrait.state === TraitState.RESOLVED) {
+        trait = trait.toAnalyzed(priorTrait.analysis);
+        if (trait.handler.register !== undefined) {
+          trait.handler.register(record.node, trait.analysis);
+        }
+      } else if (priorTrait.state === TraitState.SKIPPED) {
+        trait = trait.toSkipped();
+      } else if (priorTrait.state === TraitState.ERRORED) {
+        trait = trait.toErrored(priorTrait.diagnostics);
+      }
+
+      record.traits.push(trait);
+    }
+
+    this.classes.set(record.node, record);
+    const sf = record.node.getSourceFile();
+    if (!this.fileToClasses.has(sf)) {
+      this.fileToClasses.set(sf, new Set<ClassDeclaration>());
+    }
+    this.fileToClasses.get(sf) !.add(record.node);
+  }
+
   private scanClassForTraits(clazz: ClassDeclaration): ClassRecord|null {
     if (!this.compileNonExportedClasses && !isExported(clazz)) {
       return null;