cargdev/angular
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

perf(ngcc): read dependencies from entry-point manifest (#36486)

Browse Source 
Previously, even if an entry-point did not need to be processed,
ngcc would always parse the files of the entry-point to compute
its dependencies. This can take a lot of time for large node_modules.

Now these dependencies are cached in the entry-point manifest,
and read from there rather than computing them every time.

See https://github.com/angular/angular/issues/36414\#issuecomment-608401834
FW-2047

PR Close #36486
This commit is contained in:
Pete Bacon Darwin
2020-04-07 17:47:46 +01:00
committed by atscott
parent 468cf69c55
commit 918e628f9b
8 changed files with 203 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/compiler-cli/ngcc/src/dependencies/dependency_host.ts
View File

@ -6,6 +6,7 @@
* found in the LICENSE file at https://angular.io/license
*/
import {AbsoluteFsPath, FileSystem, PathSegment} from '../../../src/ngtsc/file_system';
import {EntryPoint} from '../packages/entry_point';
import {resolveFileWithPostfixes} from '../utils';
import {ModuleResolver} from './module_resolver';
@ -21,6 +22,11 @@ export interface DependencyInfo {
deepImports: Set<AbsoluteFsPath>;
}
export interface EntryPointWithDependencies {
entryPoint: EntryPoint;
depInfo: DependencyInfo;
}
export function createDependencyInfo(): DependencyInfo {
return {dependencies: new Set(), missing: new Set(), deepImports: new Set()};
}

39
packages/compiler-cli/ngcc/src/dependencies/dependency_resolver.ts
View File

@ -14,7 +14,7 @@ import {NgccConfiguration} from '../packages/configuration';
import {EntryPoint, EntryPointFormat, getEntryPointFormat, SUPPORTED_FORMAT_PROPERTIES} from '../packages/entry_point';
import {PartiallyOrderedList} from '../utils';
import {createDependencyInfo, DependencyHost, DependencyInfo} from './dependency_host';
import {createDependencyInfo, DependencyHost, EntryPointWithDependencies} from './dependency_host';
const builtinNodeJsModules = new Set<string>(require('module').builtinModules);
@ -94,7 +94,7 @@ export class DependencyResolver {
* @param target If provided, only return entry-points depended on by this entry-point.
* @returns the result of sorting the entry points by dependency.
*/
sortEntryPointsByDependency(entryPoints: EntryPoint[], target?: EntryPoint):
sortEntryPointsByDependency(entryPoints: EntryPointWithDependencies[], target?: EntryPoint):
SortedEntryPointsInfo {
const {invalidEntryPoints, ignoredDependencies, graph} =
this.computeDependencyGraph(entryPoints);
@ -120,18 +120,21 @@ export class DependencyResolver {
};
}
getEntryPointDependencies(entryPoint: EntryPoint): DependencyInfo {
const formatInfo = this.getEntryPointFormatInfo(entryPoint);
const host = this.hosts[formatInfo.format];
if (!host) {
throw new Error(
`Could not find a suitable format for computing dependencies of entry-point: '${
entryPoint.path}'.`);
getEntryPointWithDependencies(entryPoint: EntryPoint): EntryPointWithDependencies {
const dependencies = createDependencyInfo();
if (entryPoint.compiledByAngular) {
// Only bother to compute dependencies of entry-points that have been compiled by Angular
const formatInfo = this.getEntryPointFormatInfo(entryPoint);
const host = this.hosts[formatInfo.format];
if (!host) {
throw new Error(
`Could not find a suitable format for computing dependencies of entry-point: '${
entryPoint.path}'.`);
}
host.collectDependencies(formatInfo.path, dependencies);
this.typingsHost.collectDependencies(entryPoint.typings, dependencies);
}
const depInfo = createDependencyInfo();
host.collectDependencies(formatInfo.path, depInfo);
this.typingsHost.collectDependencies(entryPoint.typings, depInfo);
return depInfo;
return {entryPoint, depInfo: dependencies};
}
/**
@ -140,20 +143,18 @@ export class DependencyResolver {
* The graph only holds entry-points that ngcc cares about and whose dependencies
* (direct and transitive) all exist.
*/
private computeDependencyGraph(entryPoints: EntryPoint[]): DependencyGraph {
private computeDependencyGraph(entryPoints: EntryPointWithDependencies[]): DependencyGraph {
const invalidEntryPoints: InvalidEntryPoint[] = [];
const ignoredDependencies: IgnoredDependency[] = [];
const graph = new DepGraph<EntryPoint>();
const angularEntryPoints = entryPoints.filter(entryPoint => entryPoint.compiledByAngular);
const angularEntryPoints = entryPoints.filter(e => e.entryPoint.compiledByAngular);
// Add the Angular compiled entry points to the graph as nodes
angularEntryPoints.forEach(entryPoint => graph.addNode(entryPoint.path, entryPoint));
angularEntryPoints.forEach(e => graph.addNode(e.entryPoint.path, e.entryPoint));
// Now add the dependencies between them
angularEntryPoints.forEach(entryPoint => {
const {dependencies, missing, deepImports} = this.getEntryPointDependencies(entryPoint);
angularEntryPoints.forEach(({entryPoint, depInfo: {dependencies, missing, deepImports}}) => {
const missingDependencies = Array.from(missing).filter(dep => !builtinNodeJsModules.has(dep));
if (missingDependencies.length > 0 && !entryPoint.ignoreMissingDependencies) {

25
packages/compiler-cli/ngcc/src/entry_point_finder/directory_walker_entry_point_finder.ts
View File

@ -6,10 +6,11 @@
* found in the LICENSE file at https://angular.io/license
*/
import {AbsoluteFsPath, FileSystem, PathSegment} from '../../../src/ngtsc/file_system';
import {EntryPointWithDependencies} from '../dependencies/dependency_host';
import {DependencyResolver, SortedEntryPointsInfo} from '../dependencies/dependency_resolver';
import {Logger} from '../logging/logger';
import {NgccConfiguration} from '../packages/configuration';
import {EntryPoint, getEntryPointInfo, INCOMPATIBLE_ENTRY_POINT, NO_ENTRY_POINT} from '../packages/entry_point';
import {getEntryPointInfo, INCOMPATIBLE_ENTRY_POINT, NO_ENTRY_POINT} from '../packages/entry_point';
import {EntryPointManifest} from '../packages/entry_point_manifest';
import {PathMappings} from '../utils';
import {NGCC_DIRECTORY} from '../writing/new_entry_point_file_writer';
@ -32,11 +33,11 @@ export class DirectoryWalkerEntryPointFinder implements EntryPointFinder {
* all package entry-points.
*/
findEntryPoints(): SortedEntryPointsInfo {
const unsortedEntryPoints: EntryPoint[] = [];
const unsortedEntryPoints: EntryPointWithDependencies[] = [];
for (const basePath of this.basePaths) {
const entryPoints = this.entryPointManifest.readEntryPointsUsingManifest(basePath) ||
this.walkBasePathForPackages(basePath);
unsortedEntryPoints.push(...entryPoints);
entryPoints.forEach(e => unsortedEntryPoints.push(e));
}
return this.resolver.sortEntryPointsByDependency(unsortedEntryPoints);
}
@ -47,10 +48,10 @@ export class DirectoryWalkerEntryPointFinder implements EntryPointFinder {
* @param basePath The path at which to start the search
* @returns an array of `EntryPoint`s that were found within `basePath`.
*/
walkBasePathForPackages(basePath: AbsoluteFsPath): EntryPoint[] {
walkBasePathForPackages(basePath: AbsoluteFsPath): EntryPointWithDependencies[] {
this.logger.debug(
`No manifest found for ${basePath} so walking the directories for entry-points.`);
const entryPoints: EntryPoint[] = trackDuration(
const entryPoints = trackDuration(
() => this.walkDirectoryForPackages(basePath),
duration => this.logger.debug(`Walking ${basePath} for entry-points took ${duration}s.`));
this.entryPointManifest.writeEntryPointManifest(basePath, entryPoints);
@ -64,7 +65,7 @@ export class DirectoryWalkerEntryPointFinder implements EntryPointFinder {
* @param sourceDirectory An absolute path to the root directory where searching begins.
* @returns an array of `EntryPoint`s that were found within `sourceDirectory`.
*/
walkDirectoryForPackages(sourceDirectory: AbsoluteFsPath): EntryPoint[] {
walkDirectoryForPackages(sourceDirectory: AbsoluteFsPath): EntryPointWithDependencies[] {
// Try to get a primary entry point from this directory
const primaryEntryPoint =
getEntryPointInfo(this.fs, this.config, this.logger, sourceDirectory, sourceDirectory);
@ -76,15 +77,15 @@ export class DirectoryWalkerEntryPointFinder implements EntryPointFinder {
return [];
}
const entryPoints: EntryPoint[] = [];
const entryPoints: EntryPointWithDependencies[] = [];
if (primaryEntryPoint !== NO_ENTRY_POINT) {
entryPoints.push(primaryEntryPoint);
entryPoints.push(this.resolver.getEntryPointWithDependencies(primaryEntryPoint));
this.collectSecondaryEntryPoints(
entryPoints, sourceDirectory, sourceDirectory, this.fs.readdir(sourceDirectory));
// Also check for any nested node_modules in this package but only if at least one of the
// entry-points was compiled by Angular.
if (entryPoints.some(e => e.compiledByAngular)) {
if (entryPoints.some(e => e.entryPoint.compiledByAngular)) {
const nestedNodeModulesPath = this.fs.join(sourceDirectory, 'node_modules');
if (this.fs.exists(nestedNodeModulesPath)) {
entryPoints.push(...this.walkDirectoryForPackages(nestedNodeModulesPath));
@ -125,8 +126,8 @@ export class DirectoryWalkerEntryPointFinder implements EntryPointFinder {
* @param paths The paths contained in the current `directory`.
*/
private collectSecondaryEntryPoints(
entryPoints: EntryPoint[], packagePath: AbsoluteFsPath, directory: AbsoluteFsPath,
paths: PathSegment[]): void {
entryPoints: EntryPointWithDependencies[], packagePath: AbsoluteFsPath,
directory: AbsoluteFsPath, paths: PathSegment[]): void {
for (const path of paths) {
if (isIgnorablePath(path)) {
// Ignore hidden files, node_modules and ngcc directory
@ -153,7 +154,7 @@ export class DirectoryWalkerEntryPointFinder implements EntryPointFinder {
const subEntryPoint =
getEntryPointInfo(this.fs, this.config, this.logger, packagePath, possibleEntryPointPath);
if (subEntryPoint !== NO_ENTRY_POINT && subEntryPoint !== INCOMPATIBLE_ENTRY_POINT) {
entryPoints.push(subEntryPoint);
entryPoints.push(this.resolver.getEntryPointWithDependencies(subEntryPoint));
isEntryPoint = true;
}

11
packages/compiler-cli/ngcc/src/entry_point_finder/targeted_entry_point_finder.ts
View File

@ -6,6 +6,7 @@
* found in the LICENSE file at https://angular.io/license
*/
import {AbsoluteFsPath, FileSystem, join, PathSegment, relative, relativeFrom} from '../../../src/ngtsc/file_system';
import {EntryPointWithDependencies} from '../dependencies/dependency_host';
import {DependencyResolver, SortedEntryPointsInfo} from '../dependencies/dependency_resolver';
import {Logger} from '../logging/logger';
import {hasBeenProcessed} from '../packages/build_marker';
@ -25,7 +26,7 @@ import {getBasePaths} from './utils';
*/
export class TargetedEntryPointFinder implements EntryPointFinder {
private unprocessedPaths: AbsoluteFsPath[] = [];
private unsortedEntryPoints = new Map<AbsoluteFsPath, EntryPoint>();
private unsortedEntryPoints = new Map<AbsoluteFsPath, EntryPointWithDependencies>();
private basePaths = getBasePaths(this.logger, this.basePath, this.pathMappings);
constructor(
@ -40,7 +41,7 @@ export class TargetedEntryPointFinder implements EntryPointFinder {
}
const targetEntryPoint = this.unsortedEntryPoints.get(this.targetPath);
const entryPoints = this.resolver.sortEntryPointsByDependency(
Array.from(this.unsortedEntryPoints.values()), targetEntryPoint);
Array.from(this.unsortedEntryPoints.values()), targetEntryPoint?.entryPoint);
const invalidTarget =
entryPoints.invalidEntryPoints.find(i => i.entryPoint.path === this.targetPath);
@ -82,9 +83,9 @@ export class TargetedEntryPointFinder implements EntryPointFinder {
if (entryPoint === null || !entryPoint.compiledByAngular) {
return;
}
this.unsortedEntryPoints.set(entryPoint.path, entryPoint);
const deps = this.resolver.getEntryPointDependencies(entryPoint);
deps.dependencies.forEach(dep => {
const entryPointWithDeps = this.resolver.getEntryPointWithDependencies(entryPoint);
this.unsortedEntryPoints.set(entryPoint.path, entryPointWithDeps);
entryPointWithDeps.depInfo.dependencies.forEach(dep => {
if (!this.unsortedEntryPoints.has(dep)) {
this.unprocessedPaths.push(dep);
}

40
packages/compiler-cli/ngcc/src/packages/entry_point_manifest.ts
View File

@ -7,12 +7,13 @@
*/
import {createHash} from 'crypto';
import {AbsoluteFsPath, FileSystem} from '../../../src/ngtsc/file_system';
import {AbsoluteFsPath, FileSystem, PathSegment} from '../../../src/ngtsc/file_system';
import {EntryPointWithDependencies} from '../dependencies/dependency_host';
import {Logger} from '../logging/logger';
import {NGCC_VERSION} from './build_marker';
import {NgccConfiguration} from './configuration';
import {EntryPoint, getEntryPointInfo, INCOMPATIBLE_ENTRY_POINT, NO_ENTRY_POINT} from './entry_point';
import {getEntryPointInfo, INCOMPATIBLE_ENTRY_POINT, NO_ENTRY_POINT} from './entry_point';
/**
* Manages reading and writing a manifest file that contains a list of all the entry-points that
@ -40,7 +41,7 @@ export class EntryPointManifest {
* @returns an array of entry-point information for all entry-points found below the given
* `basePath` or `null` if the manifest was out of date.
*/
readEntryPointsUsingManifest(basePath: AbsoluteFsPath): EntryPoint[]|null {
readEntryPointsUsingManifest(basePath: AbsoluteFsPath): EntryPointWithDependencies[]|null {
try {
if (this.fs.basename(basePath) !== 'node_modules') {
return null;
@ -67,8 +68,9 @@ export class EntryPointManifest {
basePath} so loading entry-point information directly.`);
const startTime = Date.now();
const entryPoints: EntryPoint[] = [];
for (const [packagePath, entryPointPath] of entryPointPaths) {
const entryPoints: EntryPointWithDependencies[] = [];
for (const [packagePath, entryPointPath, dependencyPaths, missingPaths, deepImportPaths] of
entryPointPaths) {
const result =
getEntryPointInfo(this.fs, this.config, this.logger, packagePath, entryPointPath);
if (result === NO_ENTRY_POINT || result === INCOMPATIBLE_ENTRY_POINT) {
@ -76,7 +78,14 @@ export class EntryPointManifest {
manifestPath} contained an invalid pair of package paths: [${packagePath}, ${
entryPointPath}]`);
} else {
entryPoints.push(result);
entryPoints.push({
entryPoint: result,
depInfo: {
dependencies: new Set(dependencyPaths),
missing: new Set(missingPaths),
deepImports: new Set(deepImportPaths),
}
});
}
}
const duration = Math.round((Date.now() - startTime) / 100) / 10;
@ -99,7 +108,8 @@ export class EntryPointManifest {
* @param basePath The path where the manifest file is to be written.
* @param entryPoints A collection of entry-points to record in the manifest.
*/
writeEntryPointManifest(basePath: AbsoluteFsPath, entryPoints: EntryPoint[]): void {
writeEntryPointManifest(basePath: AbsoluteFsPath, entryPoints: EntryPointWithDependencies[]):
void {
if (this.fs.basename(basePath) !== 'node_modules') {
return;
}
@ -112,7 +122,14 @@ export class EntryPointManifest {
ngccVersion: NGCC_VERSION,
configFileHash: this.config.hash,
lockFileHash: lockFileHash,
entryPointPaths: entryPoints.map(entryPoint => [entryPoint.package, entryPoint.path]),
entryPointPaths: entryPoints.map(
e =>
[e.entryPoint.package,
e.entryPoint.path,
Array.from(e.depInfo.dependencies),
Array.from(e.depInfo.missing),
Array.from(e.depInfo.deepImports),
]),
};
this.fs.writeFile(this.getEntryPointManifestPath(basePath), JSON.stringify(manifest));
}
@ -143,7 +160,7 @@ export class EntryPointManifest {
* called.
*/
export class InvalidatingEntryPointManifest extends EntryPointManifest {
readEntryPointsUsingManifest(basePath: AbsoluteFsPath): EntryPoint[]|null {
readEntryPointsUsingManifest(_basePath: AbsoluteFsPath): EntryPointWithDependencies[]|null {
return null;
}
}
@ -155,5 +172,8 @@ export interface EntryPointManifestFile {
ngccVersion: string;
configFileHash: string;
lockFileHash: string;
entryPointPaths: Array<[AbsoluteFsPath, AbsoluteFsPath]>;
entryPointPaths: Array<[
AbsoluteFsPath, AbsoluteFsPath, AbsoluteFsPath[], (AbsoluteFsPath | PathSegment)[],
AbsoluteFsPath[]
]>;
}