cargdev/angular
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(compiler): skip type check and emit in bazel in some cases. (#19646)

Browse Source 
If no user files changed:
- only type check the changed generated files

Never emit non changed generated files
- we still calculate them, but don’t send them through
  TypeScript to emit them but cache the written files instead.
PR Close #19646
This commit is contained in:
Tobias Bosch
2017-10-10 13:45:42 -07:00
committed by Chuck Jazdzewski
parent 3acf9c7063
commit a22121d65d
21 changed files with 596 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/bazel/src/ngc-wrapped/BUILD.bazel
View File

@ -7,6 +7,7 @@ ts_library(
name = "ngc_lib",
srcs = [
"index.ts",
"emit_cache.ts",
"extract_i18n.ts",
],
deps = [

117
packages/bazel/src/ngc-wrapped/emit_cache.ts Normal file
View File

@ -0,0 +1,117 @@
/**
* @license
* Copyright Google Inc. All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import * as ng from '@angular/compiler-cli';
import {CompilerHost, debug, fixUmdModuleDeclarations} from '@bazel/typescript';
import * as tsickle from 'tsickle';
import * as ts from 'typescript';
interface EmitCacheEntry {
emitResult: tsickle.EmitResult;
writtenFiles: Array<{fileName: string, content: string, sourceFiles?: ts.SourceFile[]}>;
generatedFile: ng.GeneratedFile;
}
interface SourceFileWithEmitCache extends ts.SourceFile {
emitCache: Map<string, EmitCacheEntry>;
}
function getCache(sf: ts.SourceFile, genFileName?: string): EmitCacheEntry|undefined {
const emitCache = (sf as SourceFileWithEmitCache).emitCache;
return emitCache ? emitCache.get(genFileName || sf.fileName) : undefined;
}
function setCache(sf: ts.SourceFile, entry: EmitCacheEntry) {
let emitCache = (sf as SourceFileWithEmitCache).emitCache;
if (!emitCache) {
emitCache = new Map();
(sf as SourceFileWithEmitCache).emitCache = emitCache;
}
emitCache.set(entry.generatedFile ? entry.generatedFile.genFileName : sf.fileName, entry);
}
export function getCachedGeneratedFile(sf: ts.SourceFile, genFileName: string): ng.GeneratedFile|
undefined {
const cacheEntry = getCache(sf, genFileName);
return cacheEntry ? cacheEntry.generatedFile : undefined;
}
export function emitWithCache(
program: ng.Program, inputsChanged: boolean, targetFileNames: string[],
compilerOpts: ng.CompilerOptions, host: CompilerHost): tsickle.EmitResult {
const emitCallback: ng.EmitCallback = ({
targetSourceFiles,
writeFile,
cancellationToken,
emitOnlyDtsFiles,
customTransformers = {}
}) => {
if (!targetSourceFiles) {
// Note: we know that we always have targetSourceFiles
// as we called `ng.Program.emit` with `targetFileNames`.
throw new Error('Unexpected state: no targetSourceFiles!');
}
let cacheHits = 0;
const mergedEmitResult = tsickle.mergeEmitResults(targetSourceFiles.map(targetSourceFile => {
const targetGeneratedFile = program.getGeneratedFile(targetSourceFile.fileName);
const cacheSf = targetGeneratedFile ?
program.getTsProgram().getSourceFile(targetGeneratedFile.srcFileName) :
targetSourceFile;
const cacheEntry = getCache(cacheSf, targetGeneratedFile && targetGeneratedFile.genFileName);
if (cacheEntry) {
let useEmitCache = false;
if (targetGeneratedFile && !program.hasChanged(targetSourceFile.fileName)) {
// we emitted a GeneratedFile with the same content as before -> use the cache
useEmitCache = true;
} else if (!inputsChanged && !targetGeneratedFile) {
// this is an input and no inputs have changed -> use the cache
useEmitCache = true;
}
if (useEmitCache) {
cacheHits++;
cacheEntry.writtenFiles.forEach(
({fileName, content, sourceFiles}) => writeFile(
fileName, content, /*writeByteOrderMark*/ false, /*onError*/ undefined,
sourceFiles));
return cacheEntry.emitResult;
}
}
const writtenFiles:
Array<{fileName: string, content: string, sourceFiles?: ts.SourceFile[]}> = [];
const recordingWriteFile =
(fileName: string, content: string, writeByteOrderMark: boolean,
onError?: (message: string) => void, sourceFiles?: ts.SourceFile[]) => {
writtenFiles.push({fileName, content, sourceFiles});
writeFile(fileName, content, writeByteOrderMark, onError, sourceFiles);
};
const emitResult = tsickle.emitWithTsickle(
program.getTsProgram(), host, host, compilerOpts, targetSourceFile, recordingWriteFile,
cancellationToken, emitOnlyDtsFiles, {
beforeTs: customTransformers.before,
afterTs: [
...(customTransformers.after || []),
fixUmdModuleDeclarations((sf: ts.SourceFile) => host.amdModuleName(sf)),
],
});
setCache(cacheSf, {
emitResult,
writtenFiles,
generatedFile: targetGeneratedFile,
});
return emitResult;
}));
debug(`Emitted ${targetSourceFiles.length} files with ${cacheHits} cache hits`);
return mergedEmitResult;
};
return program
.emit({
targetFileNames,
emitCallback,
emitFlags: ng.EmitFlags.DTS | ng.EmitFlags.JS | ng.EmitFlags.Codegen
}) as tsickle.EmitResult;
}

2
packages/bazel/src/ngc-wrapped/extract_i18n.ts
View File

@ -11,6 +11,6 @@
// Entry point
if (require.main === module) {
const args = process.argv.slice(2);
console.error('>>> now yet implemented!');
console.error('>>> not yet implemented!');
process.exitCode = 1;
}

104
packages/bazel/src/ngc-wrapped/index.ts
View File

@ -8,12 +8,14 @@
// TODO(tbosch): figure out why we need this as it breaks node code within ngc-wrapped
/// <reference types="node" />
import * as ng from '@angular/compiler-cli';
import {BazelOptions, CachedFileLoader, CompilerHost, FileCache, FileLoader, UncachedFileLoader, constructManifest, debug, fixUmdModuleDeclarations, parseTsconfig, runAsWorker, runWorkerLoop} from '@bazel/typescript';
import {BazelOptions, CachedFileLoader, CompilerHost, FileCache, FileLoader, UncachedFileLoader, constructManifest, debug, parseTsconfig, runAsWorker, runWorkerLoop} from '@bazel/typescript';
import * as fs from 'fs';
import * as path from 'path';
import * as tsickle from 'tsickle';
import * as ts from 'typescript';
import {emitWithCache, getCachedGeneratedFile} from './emit_cache';
const EXT = /(\.ts|\.d\.ts|\.js|\.jsx|\.tsx)$/;
const NGC_GEN_FILES = /^(.*?)\.(ngfactory|ngsummary|ngstyle|shim\.ngstyle)(.*)$/;
// FIXME: we should be able to add the assets to the tsconfig so FileLoader
@ -74,23 +76,32 @@ export function relativeToRootDirs(filePath: string, rootDirs: string[]): string
}
export function compile({allowNonHermeticReads, allDepsCompiledWithBazel = true, compilerOpts,
tsHost, bazelOpts, files, inputs, expectedOuts, gatherDiagnostics}: {
tsHost, bazelOpts, files, inputs, expectedOuts,
gatherDiagnostics = defaultGatherDiagnostics}: {
allowNonHermeticReads: boolean,
allDepsCompiledWithBazel?: boolean,
compilerOpts: ng.CompilerOptions,
tsHost: ts.CompilerHost, inputs?: {[path: string]: string},
bazelOpts: BazelOptions,
files: string[],
expectedOuts: string[], gatherDiagnostics?: (program: ng.Program) => ng.Diagnostics
expectedOuts: string[],
gatherDiagnostics?: (program: ng.Program, inputsToCheck: ts.SourceFile[],
genFilesToCheck: ng.GeneratedFile[]) => ng.Diagnostics
}): {diagnostics: ng.Diagnostics, program: ng.Program} {
let fileLoader: FileLoader;
const oldFiles = new Map<string, ts.SourceFile>();
if (inputs) {
fileLoader = new CachedFileLoader(fileCache, allowNonHermeticReads);
// Resolve the inputs to absolute paths to match TypeScript internals
const resolvedInputs: {[path: string]: string} = {};
for (const key of Object.keys(inputs)) {
resolvedInputs[path.resolve(key)] = inputs[key];
const resolvedKey = path.resolve(key);
resolvedInputs[resolvedKey] = inputs[key];
const cachedSf = fileCache.getCache(resolvedKey);
if (cachedSf) {
oldFiles.set(resolvedKey, cachedSf);
}
}
fileCache.updateCache(resolvedInputs);
} else {
@ -178,40 +189,56 @@ export function compile({allowNonHermeticReads, allDepsCompiledWithBazel = true,
path.resolve(bazelBin, fileName) + '.d.ts';
}
const emitCallback: ng.TsEmitCallback = ({
program,
targetSourceFile,
writeFile,
cancellationToken,
emitOnlyDtsFiles,
customTransformers = {},
}) =>
tsickle.emitWithTsickle(
program, bazelHost, bazelHost, compilerOpts, targetSourceFile, writeFile,
cancellationToken, emitOnlyDtsFiles, {
beforeTs: customTransformers.before,
afterTs: [
...(customTransformers.after || []),
fixUmdModuleDeclarations((sf: ts.SourceFile) => bazelHost.amdModuleName(sf)),
],
});
const oldProgram = {
getSourceFile: (fileName: string) => { return oldFiles.get(fileName); },
getGeneratedFile: (srcFileName: string, genFileName: string) => {
const sf = oldFiles.get(srcFileName);
return sf ? getCachedGeneratedFile(sf, genFileName) : undefined;
},
};
const program =
ng.createProgram({rootNames: files, host: ngHost, options: compilerOpts, oldProgram});
let inputsChanged = files.some(fileName => program.hasChanged(fileName));
if (!gatherDiagnostics) {
gatherDiagnostics = (program) =>
gatherDiagnosticsForInputsOnly(compilerOpts, bazelOpts, program);
let genFilesToCheck: ng.GeneratedFile[];
let inputsToCheck: ts.SourceFile[];
if (inputsChanged) {
// if an input file changed, we need to type check all
// of our compilation sources as well as all generated files.
inputsToCheck = bazelOpts.compilationTargetSrc.map(
fileName => program.getTsProgram().getSourceFile(fileName));
genFilesToCheck = program.getGeneratedFiles().filter(gf => gf.genFileName.endsWith('.ts'));
} else {
// if no input file changed, only type check the changed generated files
// as these don't influence each other nor the type check of the input files.
inputsToCheck = [];
genFilesToCheck = program.getGeneratedFiles().filter(
gf => program.hasChanged(gf.genFileName) && gf.genFileName.endsWith('.ts'));
}
debug(
`TypeChecking ${inputsToCheck ? inputsToCheck.length : 'all'} inputs and ${genFilesToCheck ? genFilesToCheck.length : 'all'} generated files`);
const diagnostics = [...gatherDiagnostics(program !, inputsToCheck, genFilesToCheck)];
let emitResult: tsickle.EmitResult|undefined;
if (!diagnostics.length) {
const targetFileNames = [...bazelOpts.compilationTargetSrc];
for (const genFile of program.getGeneratedFiles()) {
if (genFile.genFileName.endsWith('.ts')) {
targetFileNames.push(genFile.genFileName);
}
}
emitResult = emitWithCache(program, inputsChanged, targetFileNames, compilerOpts, bazelHost);
diagnostics.push(...emitResult.diagnostics);
}
const {diagnostics, emitResult, program} = ng.performCompilation(
{rootNames: files, options: compilerOpts, host: ngHost, emitCallback, gatherDiagnostics});
const tsickleEmitResult = emitResult as tsickle.EmitResult;
let externs = '/** @externs */\n';
if (diagnostics.length) {
console.error(ng.formatDiagnostics(compilerOpts, diagnostics));
} else {
} else if (emitResult) {
if (bazelOpts.tsickleGenerateExterns) {
externs += tsickle.getGeneratedExterns(tsickleEmitResult.externs);
externs += tsickle.getGeneratedExterns(emitResult.externs);
}
if (bazelOpts.manifest) {
const manifest = constructManifest(tsickleEmitResult.modulesManifest, bazelHost);
const manifest = constructManifest(emitResult.modulesManifest, bazelHost);
fs.writeFileSync(bazelOpts.manifest, manifest);
}
}
@ -230,14 +257,9 @@ export function compile({allowNonHermeticReads, allDepsCompiledWithBazel = true,
return {program, diagnostics};
}
function isCompilationTarget(bazelOpts: BazelOptions, sf: ts.SourceFile): boolean {
return !NGC_GEN_FILES.test(sf.fileName) &&
(bazelOpts.compilationTargetSrc.indexOf(sf.fileName) !== -1);
}
function gatherDiagnosticsForInputsOnly(
options: ng.CompilerOptions, bazelOpts: BazelOptions,
ngProgram: ng.Program): (ng.Diagnostic | ts.Diagnostic)[] {
function defaultGatherDiagnostics(
ngProgram: ng.Program, inputsToCheck: ts.SourceFile[],
genFilesToCheck: ng.GeneratedFile[]): (ng.Diagnostic | ts.Diagnostic)[] {
const tsProgram = ngProgram.getTsProgram();
const diagnostics: (ng.Diagnostic | ts.Diagnostic)[] = [];
// These checks mirror ts.getPreEmitDiagnostics, with the important
@ -245,7 +267,7 @@ function gatherDiagnosticsForInputsOnly(
// program.getDeclarationDiagnostics() it somehow corrupts the emit.
diagnostics.push(...tsProgram.getOptionsDiagnostics());
diagnostics.push(...tsProgram.getGlobalDiagnostics());
for (const sf of tsProgram.getSourceFiles().filter(f => isCompilationTarget(bazelOpts, f))) {
for (const sf of inputsToCheck) {
// Note: We only get the diagnostics for individual files
// to e.g. not check libraries.
diagnostics.push(...tsProgram.getSyntacticDiagnostics(sf));
@ -255,7 +277,9 @@ function gatherDiagnosticsForInputsOnly(
// only gather the angular diagnostics if we have no diagnostics
// in any other files.
diagnostics.push(...ngProgram.getNgStructuralDiagnostics());
diagnostics.push(...ngProgram.getNgSemanticDiagnostics());
for (const genFile of genFilesToCheck) {
diagnostics.push(...ngProgram.getNgSemanticDiagnostics(genFile));
}
}
return diagnostics;
}