From fef3d2ad53f301d134ea167e51fa74d987148bd0 Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Tue, 29 Aug 2017 22:09:55 -0700 Subject: [PATCH] build: ngc-wrapped as a bazel worker (#18960) PR Close #18960 --- .bazelrc | 18 +++++++-- BUILD.bazel | 37 ++++++++++-------- integration/bazel/package.json | 1 - npm-shrinkwrap.clean.json | 2 +- npm-shrinkwrap.json | 6 +-- packages/bazel/package.json | 2 +- packages/bazel/src/ng_module.bzl | 3 +- packages/bazel/src/ngc-wrapped/BUILD.bazel | 6 ++- packages/bazel/src/ngc-wrapped/index.ts | 44 ++++++++++++++++++---- 9 files changed, 84 insertions(+), 35 deletions(-) diff --git a/.bazelrc b/.bazelrc index 7b35f14ca3..026b12684f 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,6 +1,18 @@ -# Disable sandboxing because it's too slow. -# https://github.com/bazelbuild/bazel/issues/2424 -build --spawn_strategy=standalone +# Make compilation fast, by keeping a few copies of the compilers +# running as daemons, and cache SourceFile AST's to reduce parse time. +build --strategy=TypeScriptCompile=worker +build --strategy=AngularTemplateCompile=worker + +# Don't create bazel-* symlinks in the WORKSPACE directory. +# These require .gitignore and may scare users. +# Also, it's a workaround for https://github.com/bazelbuild/rules_typescript/issues/12 +# which affects the common case of having `tsconfig.json` in the WORKSPACE directory. +# +# Instead, you should run `bazel info bazel-bin` to find out where the outputs went. +build --symlink_prefix=/ # Performance: avoid stat'ing input files build --watchfs + +# Don't print all the .d.ts output locations after builds +build --show_result=0 \ No newline at end of file diff --git a/BUILD.bazel b/BUILD.bazel index 3ad279d7aa..718cd4f758 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -6,20 +6,25 @@ exports_files(["tsconfig.json"]) # https://github.com/bazelbuild/bazel/issues/374#issuecomment-296217940 filegroup( name = "node_modules", - srcs = glob([ - # Performance workaround: list individual files - # This won't scale in the general case. - # TODO(alexeagle): figure out what to do - "node_modules/typescript/**", - "node_modules/zone.js/**", - "node_modules/rxjs/**/*.d.ts", - "node_modules/rxjs/**/*.js", - "node_modules/@types/**/*.d.ts", - "node_modules/tsickle/**", - "node_modules/hammerjs/**/*.d.ts", - "node_modules/protobufjs/**", - "node_modules/bytebuffer/**", - "node_modules/reflect-metadata/**", - "node_modules/minimist/**/*.js", - ]), + # Performance workaround: list individual files + # Reduces the number of files as inputs to nodejs_binary: + # bazel query "deps(:node_modules)" | wc -l + # This won't scale in the general case. + # TODO(alexeagle): figure out what to do + srcs = glob(["/".join(["node_modules", pkg, "**", ext]) for pkg in [ + "typescript", + "zone.js", + "rxjs", + "@types", + "tsickle", + "hammerjs", + "protobufjs", + "bytebuffer", + "reflect-metadata", + "minimist", + ] for ext in [ + "*.js", + "*.json", + "*.d.ts", + ]]), ) diff --git a/integration/bazel/package.json b/integration/bazel/package.json index 4579e0f29d..54d6c510c5 100644 --- a/integration/bazel/package.json +++ b/integration/bazel/package.json @@ -15,7 +15,6 @@ "devDependencies": { "@angular/bazel": "file:../../dist/packages-dist/bazel", "@angular/compiler-cli": "file:../../dist/packages-dist/compiler-cli", - "@bazel/typescript": "0.0.7", "typescript": "~2.3.1" }, "scripts": { diff --git a/npm-shrinkwrap.clean.json b/npm-shrinkwrap.clean.json index 305f5095bf..c388799731 100644 --- a/npm-shrinkwrap.clean.json +++ b/npm-shrinkwrap.clean.json @@ -3,7 +3,7 @@ "version": "5.0.0-beta.5", "dependencies": { "@bazel/typescript": { - "version": "0.0.7", + "version": "0.0.9", "dependencies": { "@types/node": { "version": "7.0.18" diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index d1419e69ab..6f144244df 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -3,9 +3,9 @@ "version": "5.0.0-beta.5", "dependencies": { "@bazel/typescript": { - "version": "0.0.7", - "from": "@bazel/typescript@0.0.7", - "resolved": "https://registry.npmjs.org/@bazel/typescript/-/typescript-0.0.7.tgz", + "version": "0.0.9", + "from": "@bazel/typescript@latest", + "resolved": "https://registry.npmjs.org/@bazel/typescript/-/typescript-0.0.9.tgz", "dependencies": { "@types/node": { "version": "7.0.18", diff --git a/packages/bazel/package.json b/packages/bazel/package.json index e624fce853..1ffdfac502 100644 --- a/packages/bazel/package.json +++ b/packages/bazel/package.json @@ -9,7 +9,7 @@ "typescript": "~2.3" }, "dependencies": { - "@bazel/typescript": "~0.0.7" + "@bazel/typescript": "0.0.9" }, "repository": { "type": "git", diff --git a/packages/bazel/src/ng_module.bzl b/packages/bazel/src/ng_module.bzl index e21ba8b847..4681d65b90 100644 --- a/packages/bazel/src/ng_module.bzl +++ b/packages/bazel/src/ng_module.bzl @@ -173,8 +173,7 @@ NG_MODULE_ATTRIBUTES = { cfg = "host", ), - # TODO(alexeagle): enable workers for ngc - "_supports_workers": attr.bool(default = False), + "_supports_workers": attr.bool(default = True), } ng_module = rule( diff --git a/packages/bazel/src/ngc-wrapped/BUILD.bazel b/packages/bazel/src/ngc-wrapped/BUILD.bazel index 5f5d0b5fbd..96e5a721be 100644 --- a/packages/bazel/src/ngc-wrapped/BUILD.bazel +++ b/packages/bazel/src/ngc-wrapped/BUILD.bazel @@ -22,6 +22,10 @@ nodejs_binary( # Entry point assumes the user is outside this WORKSPACE, # and references our rules with @angular//src/ngc-wrapped entry_point = "angular/src/ngc-wrapped/index.js", - data = [":ngc_lib"], + args = ["--node_options=--expose-gc"], + data = [ + ":ngc_lib", + "@build_bazel_rules_typescript//internal:worker_protocol.proto" + ], visibility = ["//visibility:public"], ) \ No newline at end of file diff --git a/packages/bazel/src/ngc-wrapped/index.ts b/packages/bazel/src/ngc-wrapped/index.ts index 7326c53510..08782fb7fa 100644 --- a/packages/bazel/src/ngc-wrapped/index.ts +++ b/packages/bazel/src/ngc-wrapped/index.ts @@ -6,7 +6,7 @@ * found in the LICENSE file at https://angular.io/license */ import * as ng from '@angular/compiler-cli'; -import {CompilerHost, UncachedFileLoader, parseTsconfig} from '@bazel/typescript'; +import {CachedFileLoader, CompilerHost, FileCache, FileLoader, UncachedFileLoader, debug, parseTsconfig, runAsWorker, runWorkerLoop} from '@bazel/typescript'; import * as fs from 'fs'; import * as path from 'path'; import * as tsickle from 'tsickle'; @@ -19,6 +19,10 @@ const NGC_NON_TS_INPUTS = /(\.(ngsummary|ngstyle|ngfactory)(\.d)?\.ts|\.ngsummary\.json|\.css|\.html)$/; // FIXME should need only summary, css, html +// TODO(alexeagle): probably not needed, see +// https://github.com/bazelbuild/rules_typescript/issues/28 +const ALLOW_NON_HERMETIC_READS = true; + function topologicalSort( result: tsickle.FileMap, current: string, modulesManifest: tsickle.ModulesManifest, visiting: tsickle.FileMap) { @@ -53,7 +57,28 @@ export function constructManifest( } export function main(args) { - const project = args[1]; + if (runAsWorker(args)) { + runWorkerLoop(runOneBuild); + } else { + return runOneBuild(args) ? 0 : 1; + } + return 0; +} + +/** The one FileCache instance used in this process. */ +const fileCache = new FileCache(debug); + +function runOneBuild(args: string[], inputs?: {[path: string]: string}): boolean { + if (args[0] === '-p') args.shift(); + // Strip leading at-signs, used to indicate a params file + const project = args[0].replace(/^@+/, ''); + let fileLoader: FileLoader; + if (inputs) { + fileLoader = new CachedFileLoader(fileCache, ALLOW_NON_HERMETIC_READS); + fileCache.updateCache(inputs); + } else { + fileLoader = new UncachedFileLoader(); + } const [{options: tsOptions, bazelOpts, files, config}] = parseTsconfig(project); const {basePath} = ng.calcProjectFileAndBasePath(project); @@ -72,7 +97,7 @@ export function main(args) { // NB: the rootDirs should have been sorted longest-first for (const dir of rootDirs || []) { const rel = path.relative(dir, filePath); - if (rel.indexOf('.') != 0) return rel; + if (rel.indexOf('.') !== 0) return rel; } return filePath; } @@ -121,9 +146,14 @@ export function main(args) { } const bazelHost = new CompilerHost( - files, tsOptions, bazelOpts, tsHost, new UncachedFileLoader(), generatedFileModuleResolver); - bazelHost.allowNonHermeticRead = (filePath: string) => - NGC_NON_TS_INPUTS.test(filePath) || filePath.split(path.sep).indexOf('node_modules') != -1; + files, tsOptions, bazelOpts, tsHost, fileLoader, ALLOW_NON_HERMETIC_READS, + generatedFileModuleResolver); + // The file cache is populated by Bazel with workspace-relative filenames + // so we must relativize paths before looking them up in the cache. + const originalGetSourceFile = bazelHost.getSourceFile.bind(bazelHost); + bazelHost.getSourceFile = (fileName: string, languageVersion: ts.ScriptTarget) => { + return originalGetSourceFile(relativeToRootDirs(fileName, [tsOptions.rootDir])); + }; bazelHost.shouldSkipTsickleProcessing = (fileName: string): boolean => bazelOpts.compilationTargetSrc.indexOf(fileName) === -1 && !NGC_NON_TS_INPUTS.test(fileName); @@ -185,7 +215,7 @@ export function main(args) { originalWriteFile(missing, '', false); } - return diagnostics.some(d => d.category === ts.DiagnosticCategory.Error) ? 1 : 0; + return diagnostics.every(d => d.category !== ts.DiagnosticCategory.Error); } if (require.main === module) {