perf(ngcc): read dependencies from entry-point manifest (#36486)
Previously, even if an entry-point did not need to be processed, ngcc would always parse the files of the entry-point to compute its dependencies. This can take a lot of time for large node_modules. Now these dependencies are cached in the entry-point manifest, and read from there rather than computing them every time. See https://github.com/angular/angular/issues/36414\#issuecomment-608401834 FW-2047 PR Close #36486
This commit is contained in:
Pete Bacon Darwin
committed by
atscott
atscott
parent
4aa4e6fd03
commit
a185efbd60
@ -10,7 +10,7 @@ import {DepGraph} from 'dependency-graph';
|
||||
|
||||
import {absoluteFrom, AbsoluteFsPath, FileSystem, getFileSystem, relativeFrom} from '../../../src/ngtsc/file_system';
|
||||
import {runInEachFileSystem} from '../../../src/ngtsc/file_system/testing';
|
||||
import {DependencyInfo} from '../../src/dependencies/dependency_host';
|
||||
import {DependencyInfo, EntryPointWithDependencies} from '../../src/dependencies/dependency_host';
|
||||
import {DependencyResolver, SortedEntryPointsInfo} from '../../src/dependencies/dependency_resolver';
|
||||
import {DtsDependencyHost} from '../../src/dependencies/dts_dependency_host';
|
||||
import {EsmDependencyHost} from '../../src/dependencies/esm_dependency_host';
|
||||
@ -131,7 +131,8 @@ runInEachFileSystem(() => {
|
||||
.and.callFake(createFakeComputeDependencies(dependencies));
|
||||
spyOn(dtsHost, 'collectDependencies')
|
||||
.and.callFake(createFakeComputeDependencies(dtsDependencies));
|
||||
const result = resolver.sortEntryPointsByDependency([fifth, first, fourth, second, third]);
|
||||
const result = resolver.sortEntryPointsByDependency(
|
||||
getEntryPointsWithDeps(resolver, [fifth, first, fourth, second, third]));
|
||||
expect(result.entryPoints).toEqual([fifth, fourth, third, second, first]);
|
||||
});
|
||||
|
||||
@ -144,7 +145,8 @@ runInEachFileSystem(() => {
|
||||
[_('/first/index.d.ts')]: {resolved: [], missing: [_('/missing')]},
|
||||
[_('/second/sub/index.d.ts')]: {resolved: [], missing: []},
|
||||
}));
|
||||
const result = resolver.sortEntryPointsByDependency([first, second]);
|
||||
const result =
|
||||
resolver.sortEntryPointsByDependency(getEntryPointsWithDeps(resolver, [first, second]));
|
||||
expect(result.entryPoints).toEqual([second]);
|
||||
expect(result.invalidEntryPoints).toEqual([
|
||||
{entryPoint: first, missingDependencies: [_('/missing')]},
|
||||
@ -163,7 +165,8 @@ runInEachFileSystem(() => {
|
||||
[_('/third/index.d.ts')]: {resolved: [], missing: []},
|
||||
}));
|
||||
// Note that we will process `first` before `second`, which has the missing dependency.
|
||||
const result = resolver.sortEntryPointsByDependency([first, second, third]);
|
||||
const result = resolver.sortEntryPointsByDependency(
|
||||
getEntryPointsWithDeps(resolver, [first, second, third]));
|
||||
expect(result.entryPoints).toEqual([third]);
|
||||
expect(result.invalidEntryPoints).toEqual([
|
||||
{entryPoint: second, missingDependencies: [_('/missing')]},
|
||||
@ -183,7 +186,8 @@ runInEachFileSystem(() => {
|
||||
[_('/third/index.d.ts')]: {resolved: [], missing: []},
|
||||
}));
|
||||
// Note that we will process `first` after `second`, which has the missing dependency.
|
||||
const result = resolver.sortEntryPointsByDependency([second, first, third]);
|
||||
const result = resolver.sortEntryPointsByDependency(
|
||||
getEntryPointsWithDeps(resolver, [second, first, third]));
|
||||
expect(result.entryPoints).toEqual([third]);
|
||||
expect(result.invalidEntryPoints).toEqual([
|
||||
{entryPoint: second, missingDependencies: [_('/missing')]},
|
||||
@ -202,7 +206,8 @@ runInEachFileSystem(() => {
|
||||
[_('/sixth/index.d.ts')]: {resolved: [], missing: [_('/missing')]},
|
||||
}));
|
||||
// Note that we will process `first` after `second`, which has the missing dependency.
|
||||
const result = resolver.sortEntryPointsByDependency([sixthIgnoreMissing, first]);
|
||||
const result = resolver.sortEntryPointsByDependency(
|
||||
getEntryPointsWithDeps(resolver, [sixthIgnoreMissing, first]));
|
||||
expect(result.entryPoints).toEqual([sixthIgnoreMissing, first]);
|
||||
expect(result.invalidEntryPoints).toEqual([]);
|
||||
});
|
||||
@ -218,7 +223,8 @@ runInEachFileSystem(() => {
|
||||
[_('/second/sub/index.d.ts')]: {resolved: [first.path], missing: []},
|
||||
[_('/sixth/index.d.ts')]: {resolved: [second.path], missing: []},
|
||||
}));
|
||||
const result = resolver.sortEntryPointsByDependency([first, second, sixthIgnoreMissing]);
|
||||
const result = resolver.sortEntryPointsByDependency(
|
||||
getEntryPointsWithDeps(resolver, [first, second, sixthIgnoreMissing]));
|
||||
// sixth has no missing dependencies, but it has _invalid_ dependencies, so it's not
|
||||
// compiled.
|
||||
expect(result.entryPoints).toEqual([]);
|
||||
@ -235,7 +241,8 @@ runInEachFileSystem(() => {
|
||||
[_('/second/sub/index.d.ts')]: {resolved: [], missing: [_('/missing2')]},
|
||||
[_('/third/index.d.ts')]: {resolved: [first.path, second.path], missing: []},
|
||||
}));
|
||||
const result = resolver.sortEntryPointsByDependency([first, second, third]);
|
||||
const result = resolver.sortEntryPointsByDependency(
|
||||
getEntryPointsWithDeps(resolver, [first, second, third]));
|
||||
expect(result.entryPoints).toEqual([]);
|
||||
expect(result.invalidEntryPoints).toEqual([
|
||||
{entryPoint: first, missingDependencies: [_('/missing1')]},
|
||||
@ -251,7 +258,8 @@ runInEachFileSystem(() => {
|
||||
spyOn(dtsHost, 'collectDependencies').and.callFake(createFakeComputeDependencies({
|
||||
[_('/first/index.d.ts')]: {resolved: [], missing: []},
|
||||
}));
|
||||
const result = resolver.sortEntryPointsByDependency([first]);
|
||||
const result =
|
||||
resolver.sortEntryPointsByDependency(getEntryPointsWithDeps(resolver, [first]));
|
||||
expect(result.entryPoints).toEqual([first]);
|
||||
expect(logger.logs.warn).toEqual([[
|
||||
`Entry point 'first' contains deep imports into '${
|
||||
@ -290,7 +298,8 @@ runInEachFileSystem(() => {
|
||||
typings: _('/project/node_modules/test-package/index.d.ts'),
|
||||
} as EntryPoint;
|
||||
|
||||
const result = resolver.sortEntryPointsByDependency([testEntryPoint]);
|
||||
const result = resolver.sortEntryPointsByDependency(
|
||||
getEntryPointsWithDeps(resolver, [testEntryPoint]));
|
||||
expect(result.entryPoints).toEqual([testEntryPoint]);
|
||||
expect(logger.logs.warn).toEqual([[
|
||||
`Entry point 'test-package' contains deep imports into '${
|
||||
@ -299,14 +308,15 @@ runInEachFileSystem(() => {
|
||||
});
|
||||
|
||||
it('should error if the entry point does not have a suitable format', () => {
|
||||
expect(() => resolver.sortEntryPointsByDependency([
|
||||
expect(() => resolver.sortEntryPointsByDependency(getEntryPointsWithDeps(resolver, [
|
||||
{path: '/first', packageJson: {}, compiledByAngular: true} as EntryPoint
|
||||
])).toThrowError(`There is no appropriate source code format in '/first' entry-point.`);
|
||||
]))).toThrowError(`There is no appropriate source code format in '/first' entry-point.`);
|
||||
});
|
||||
|
||||
it('should error if there is no appropriate DependencyHost for the given formats', () => {
|
||||
resolver = new DependencyResolver(fs, new MockLogger(), config, {esm2015: host}, host);
|
||||
expect(() => resolver.sortEntryPointsByDependency([first]))
|
||||
expect(
|
||||
() => resolver.sortEntryPointsByDependency(getEntryPointsWithDeps(resolver, [first])))
|
||||
.toThrowError(
|
||||
`Could not find a suitable format for computing dependencies of entry-point: '${
|
||||
first.path}'.`);
|
||||
@ -317,7 +327,8 @@ runInEachFileSystem(() => {
|
||||
.and.callFake(createFakeComputeDependencies(dependencies));
|
||||
spyOn(dtsHost, 'collectDependencies')
|
||||
.and.callFake(createFakeComputeDependencies(dtsDependencies));
|
||||
const result = resolver.sortEntryPointsByDependency([fifth, first, fourth, second, third]);
|
||||
const result = resolver.sortEntryPointsByDependency(
|
||||
getEntryPointsWithDeps(resolver, [fifth, first, fourth, second, third]));
|
||||
expect(result.ignoredDependencies).toEqual([
|
||||
{entryPoint: first, dependencyPath: _('/ignored-1')},
|
||||
{entryPoint: third, dependencyPath: _('/ignored-2')},
|
||||
@ -329,7 +340,8 @@ runInEachFileSystem(() => {
|
||||
.and.callFake(createFakeComputeDependencies(dependencies));
|
||||
spyOn(dtsHost, 'collectDependencies')
|
||||
.and.callFake(createFakeComputeDependencies(dtsDependencies));
|
||||
const result = resolver.sortEntryPointsByDependency([fifth, first, fourth, second, third]);
|
||||
const result = resolver.sortEntryPointsByDependency(
|
||||
getEntryPointsWithDeps(resolver, [fifth, first, fourth, second, third]));
|
||||
|
||||
expect(result.graph).toEqual(jasmine.any(DepGraph));
|
||||
expect(result.graph.size()).toBe(5);
|
||||
@ -341,7 +353,7 @@ runInEachFileSystem(() => {
|
||||
.and.callFake(createFakeComputeDependencies(dependencies));
|
||||
spyOn(dtsHost, 'collectDependencies')
|
||||
.and.callFake(createFakeComputeDependencies(dtsDependencies));
|
||||
const entryPoints = [fifth, first, fourth, second, third];
|
||||
const entryPoints = getEntryPointsWithDeps(resolver, [fifth, first, fourth, second, third]);
|
||||
let sorted: SortedEntryPointsInfo;
|
||||
|
||||
sorted = resolver.sortEntryPointsByDependency(entryPoints, first);
|
||||
@ -363,7 +375,7 @@ runInEachFileSystem(() => {
|
||||
spyOn(dtsHost, 'collectDependencies').and.callFake(createFakeComputeDependencies({
|
||||
[_('/first/index.d.ts')]: {resolved: [], missing: [_('/missing')]},
|
||||
}));
|
||||
const entryPoints = [first];
|
||||
const entryPoints = getEntryPointsWithDeps(resolver, [first]);
|
||||
let sorted: SortedEntryPointsInfo;
|
||||
|
||||
sorted = resolver.sortEntryPointsByDependency(entryPoints, first);
|
||||
@ -379,7 +391,7 @@ runInEachFileSystem(() => {
|
||||
spyOn(dtsHost, 'collectDependencies').and.callFake(createFakeComputeDependencies({
|
||||
[_('/first/index.d.ts')]: {resolved: [], missing: ['fs']},
|
||||
}));
|
||||
const entryPoints = [first];
|
||||
const entryPoints = getEntryPointsWithDeps(resolver, [first]);
|
||||
let sorted: SortedEntryPointsInfo;
|
||||
|
||||
sorted = resolver.sortEntryPointsByDependency(entryPoints, first);
|
||||
@ -400,7 +412,8 @@ runInEachFileSystem(() => {
|
||||
.and.callFake(createFakeComputeDependencies(dependencies));
|
||||
spyOn(dtsHost, 'collectDependencies')
|
||||
.and.callFake(createFakeComputeDependencies(dtsDependencies));
|
||||
const result = resolver.sortEntryPointsByDependency([fifth, first, fourth, second, third]);
|
||||
const result = resolver.sortEntryPointsByDependency(
|
||||
getEntryPointsWithDeps(resolver, [fifth, first, fourth, second, third]));
|
||||
expect(result.entryPoints).toEqual([fifth, fourth, third, second, first]);
|
||||
|
||||
expect(esm5Host.collectDependencies)
|
||||
@ -449,7 +462,7 @@ runInEachFileSystem(() => {
|
||||
[_('/second/sub/index.d.ts')]: {resolved: [], missing: [_('/missing2')]},
|
||||
[_('/third/index.d.ts')]: {resolved: [second.path], missing: []},
|
||||
}));
|
||||
const entryPoints = [first, second, third];
|
||||
const entryPoints = getEntryPointsWithDeps(resolver, [first, second, third]);
|
||||
const sorted = resolver.sortEntryPointsByDependency(entryPoints);
|
||||
expect(sorted.entryPoints).toEqual([first]);
|
||||
expect(sorted.invalidEntryPoints).toEqual([
|
||||
@ -469,6 +482,11 @@ runInEachFileSystem(() => {
|
||||
return {dependencies, missing, deepImports};
|
||||
};
|
||||
}
|
||||
|
||||
function getEntryPointsWithDeps(
|
||||
resolver: DependencyResolver, entryPoints: EntryPoint[]): EntryPointWithDependencies[] {
|
||||
return entryPoints.map(entryPoint => resolver.getEntryPointWithDependencies(entryPoint));
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@ -1107,7 +1107,12 @@ runInEachFileSystem(() => {
|
||||
// had removed earlier.
|
||||
manifest = JSON.parse(fs.readFile(_('/node_modules/__ngcc_entry_points__.json')));
|
||||
expect(manifest.entryPointPaths).toContain([
|
||||
_('/node_modules/@angular/common'), _('/node_modules/@angular/common/testing')
|
||||
_('/node_modules/@angular/common'), _('/node_modules/@angular/common/testing'),
|
||||
[
|
||||
_('/node_modules/@angular/core'), _('/node_modules/@angular/common'),
|
||||
_('/node_modules/rxjs')
|
||||
],
|
||||
[], []
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@ -7,12 +7,12 @@
|
||||
*/
|
||||
import {createHash} from 'crypto';
|
||||
|
||||
import {absoluteFrom, FileSystem, getFileSystem} from '../../../src/ngtsc/file_system';
|
||||
import {absoluteFrom, FileSystem, getFileSystem, relativeFrom} from '../../../src/ngtsc/file_system';
|
||||
import {runInEachFileSystem} from '../../../src/ngtsc/file_system/testing';
|
||||
import {loadTestFiles} from '../../../test/helpers';
|
||||
import {EntryPointWithDependencies} from '../../src/dependencies/dependency_host';
|
||||
import {NGCC_VERSION} from '../../src/packages/build_marker';
|
||||
import {NgccConfiguration} from '../../src/packages/configuration';
|
||||
import {EntryPoint} from '../../src/packages/entry_point';
|
||||
import {EntryPointManifest, EntryPointManifestFile} from '../../src/packages/entry_point_manifest';
|
||||
import {MockLogger} from '../helpers/mock_logger';
|
||||
|
||||
@ -129,22 +129,48 @@ runInEachFileSystem(() => {
|
||||
]);
|
||||
manifestFile.entryPointPaths.push([
|
||||
_Abs('/project/node_modules/some_package'),
|
||||
_Abs('/project/node_modules/some_package/valid_entry_point')
|
||||
_Abs('/project/node_modules/some_package/valid_entry_point'),
|
||||
[
|
||||
_Abs('/project/node_modules/other_package_1'),
|
||||
_Abs('/project/node_modules/other_package_2'),
|
||||
],
|
||||
[
|
||||
_Abs('/project/node_modules/missing_1'),
|
||||
relativeFrom('missing_2'),
|
||||
],
|
||||
[
|
||||
_Abs('/project/node_modules/deep/import/path'),
|
||||
],
|
||||
]);
|
||||
fs.writeFile(
|
||||
_Abs('/project/node_modules/__ngcc_entry_points__.json'), JSON.stringify(manifestFile));
|
||||
const entryPoints = manifest.readEntryPointsUsingManifest(_Abs('/project/node_modules'));
|
||||
expect(entryPoints).toEqual([{
|
||||
name: 'some_package/valid_entry_point',
|
||||
packageJson: jasmine.any(Object),
|
||||
package: _Abs('/project/node_modules/some_package'),
|
||||
path: _Abs('/project/node_modules/some_package/valid_entry_point'),
|
||||
typings:
|
||||
_Abs('/project/node_modules/some_package/valid_entry_point/valid_entry_point.d.ts'),
|
||||
compiledByAngular: true,
|
||||
ignoreMissingDependencies: false,
|
||||
generateDeepReexports: false,
|
||||
} as any]);
|
||||
entryPoint: {
|
||||
name: 'some_package/valid_entry_point',
|
||||
packageJson: jasmine.any(Object),
|
||||
package: _Abs('/project/node_modules/some_package'),
|
||||
path: _Abs('/project/node_modules/some_package/valid_entry_point'),
|
||||
typings:
|
||||
_Abs('/project/node_modules/some_package/valid_entry_point/valid_entry_point.d.ts'),
|
||||
compiledByAngular: true,
|
||||
ignoreMissingDependencies: false,
|
||||
generateDeepReexports: false,
|
||||
} as any,
|
||||
depInfo: {
|
||||
dependencies: new Set([
|
||||
_Abs('/project/node_modules/other_package_1'),
|
||||
_Abs('/project/node_modules/other_package_2'),
|
||||
]),
|
||||
missing: new Set([
|
||||
_Abs('/project/node_modules/missing_1'),
|
||||
relativeFrom('missing_2'),
|
||||
]),
|
||||
deepImports: new Set([
|
||||
_Abs('/project/node_modules/deep/import/path'),
|
||||
])
|
||||
}
|
||||
}]);
|
||||
});
|
||||
|
||||
it('should return null if any of the entry-points are not valid', () => {
|
||||
@ -152,7 +178,7 @@ runInEachFileSystem(() => {
|
||||
fs.writeFile(_Abs('/project/yarn.lock'), 'LOCK FILE CONTENTS');
|
||||
manifestFile.entryPointPaths.push([
|
||||
_Abs('/project/node_modules/some_package'),
|
||||
_Abs('/project/node_modules/some_package/valid_entry_point')
|
||||
_Abs('/project/node_modules/some_package/valid_entry_point'), [], [], []
|
||||
]);
|
||||
fs.writeFile(
|
||||
_Abs('/project/node_modules/__ngcc_entry_points__.json'), JSON.stringify(manifestFile));
|
||||
@ -223,14 +249,36 @@ runInEachFileSystem(() => {
|
||||
|
||||
it('should write the package path and entry-point path of each entry-point provided', () => {
|
||||
fs.writeFile(_Abs('/project/package-lock.json'), 'LOCK FILE CONTENTS');
|
||||
const entryPoint1 = {
|
||||
package: _Abs('/project/node_modules/package-1/'),
|
||||
path: _Abs('/project/node_modules/package-1/'),
|
||||
} as unknown as EntryPoint;
|
||||
const entryPoint2 = {
|
||||
package: _Abs('/project/node_modules/package-2/'),
|
||||
path: _Abs('/project/node_modules/package-2/entry-point'),
|
||||
} as unknown as EntryPoint;
|
||||
const entryPoint1: EntryPointWithDependencies = {
|
||||
entryPoint: {
|
||||
package: _Abs('/project/node_modules/package-1/'),
|
||||
path: _Abs('/project/node_modules/package-1/'),
|
||||
} as any,
|
||||
depInfo: {
|
||||
dependencies: new Set([
|
||||
_Abs('/project/node_modules/other_package_1'),
|
||||
_Abs('/project/node_modules/other_package_2'),
|
||||
]),
|
||||
missing: new Set(),
|
||||
deepImports: new Set()
|
||||
}
|
||||
};
|
||||
const entryPoint2: EntryPointWithDependencies = {
|
||||
entryPoint: {
|
||||
package: _Abs('/project/node_modules/package-2/'),
|
||||
path: _Abs('/project/node_modules/package-2/entry-point'),
|
||||
} as any,
|
||||
depInfo: {
|
||||
dependencies: new Set(),
|
||||
missing: new Set([
|
||||
_Abs('/project/node_modules/missing_1'),
|
||||
relativeFrom('missing_2'),
|
||||
]),
|
||||
deepImports: new Set([
|
||||
_Abs('/project/node_modules/deep/import/path'),
|
||||
])
|
||||
}
|
||||
};
|
||||
manifest.writeEntryPointManifest(_Abs('/project/node_modules'), [entryPoint1, entryPoint2]);
|
||||
const file: EntryPointManifestFile =
|
||||
JSON.parse(fs.readFile(_Abs('/project/node_modules/__ngcc_entry_points__.json')));
|
||||
@ -238,10 +286,24 @@ runInEachFileSystem(() => {
|
||||
[
|
||||
_Abs('/project/node_modules/package-1/'),
|
||||
_Abs('/project/node_modules/package-1/'),
|
||||
[
|
||||
_Abs('/project/node_modules/other_package_1'),
|
||||
_Abs('/project/node_modules/other_package_2'),
|
||||
],
|
||||
[],
|
||||
[],
|
||||
],
|
||||
[
|
||||
_Abs('/project/node_modules/package-2/'),
|
||||
_Abs('/project/node_modules/package-2/entry-point'),
|
||||
[],
|
||||
[
|
||||
_Abs('/project/node_modules/missing_1'),
|
||||
relativeFrom('missing_2'),
|
||||
],
|
||||
[
|
||||
_Abs('/project/node_modules/deep/import/path'),
|
||||
],
|
||||
]
|
||||
]);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user