From ae876d1317cfc23b1dc0f02a53acd49e10338c0a Mon Sep 17 00:00:00 2001 From: Chuck Jazdzewski Date: Tue, 8 Mar 2016 10:27:54 -0800 Subject: [PATCH] feat(build): Persisting decorator metadata This allows determing what the runtime metadata will be for a class without having to loading and running the corresponding .js file. --- tools/broccoli/broccoli-typescript.ts | 39 +++- tools/metadata/evaluator.ts | 304 ++++++++++++++++++++++++++ tools/metadata/extractor.ts | 92 ++++++++ tools/metadata/symbols.ts | 34 +++ 4 files changed, 467 insertions(+), 2 deletions(-) create mode 100644 tools/metadata/evaluator.ts create mode 100644 tools/metadata/extractor.ts create mode 100644 tools/metadata/symbols.ts diff --git a/tools/broccoli/broccoli-typescript.ts b/tools/broccoli/broccoli-typescript.ts index 859b51dd10..b09847555c 100644 --- a/tools/broccoli/broccoli-typescript.ts +++ b/tools/broccoli/broccoli-typescript.ts @@ -5,7 +5,7 @@ import fse = require('fs-extra'); import path = require('path'); import * as ts from 'typescript'; import {wrapDiffingPlugin, DiffingBroccoliPlugin, DiffResult} from './diffing-broccoli-plugin'; - +import {MetadataExtractor} from '../metadata/extractor'; type FileRegistry = ts.Map<{version: number}>; @@ -50,6 +50,7 @@ class DiffingTSCompiler implements DiffingBroccoliPlugin { private rootFilePaths: string[]; private tsServiceHost: ts.LanguageServiceHost; private tsService: ts.LanguageService; + private metadataExtractor: MetadataExtractor; private firstRun: boolean = true; private previousRunFailed: boolean = false; // Whether to generate the @internal typing files (they are only generated when `stripInternal` is @@ -92,6 +93,7 @@ class DiffingTSCompiler implements DiffingBroccoliPlugin { this.tsServiceHost = new CustomLanguageServiceHost(this.tsOpts, this.rootFilePaths, this.fileRegistry, this.inputPath); this.tsService = ts.createLanguageService(this.tsServiceHost, ts.createDocumentRegistry()); + this.metadataExtractor = new MetadataExtractor(this.tsService); } @@ -124,6 +126,8 @@ class DiffingTSCompiler implements DiffingBroccoliPlugin { this.firstRun = false; this.doFullBuild(); } else { + let program = this.tsService.getProgram(); + let typeChecker = program.getTypeChecker(); tsEmitInternal = false; pathsToEmit.forEach((tsFilePath) => { let output = this.tsService.getEmitOutput(tsFilePath); @@ -139,6 +143,10 @@ class DiffingTSCompiler implements DiffingBroccoliPlugin { let destDirPath = path.dirname(o.name); fse.mkdirsSync(destDirPath); fs.writeFileSync(o.name, this.fixSourceMapSources(o.text), FS_OPTS); + if (endsWith(o.name, '.d.ts')) { + const sourceFile = program.getSourceFile(tsFilePath); + this.emitMetadata(o.name, sourceFile, typeChecker); + } }); } }); @@ -194,11 +202,22 @@ class DiffingTSCompiler implements DiffingBroccoliPlugin { private doFullBuild() { let program = this.tsService.getProgram(); - + let typeChecker = program.getTypeChecker(); + let diagnostics: ts.Diagnostic[] = []; tsEmitInternal = false; + let emitResult = program.emit(undefined, (absoluteFilePath, fileContent) => { fse.mkdirsSync(path.dirname(absoluteFilePath)); fs.writeFileSync(absoluteFilePath, this.fixSourceMapSources(fileContent), FS_OPTS); + if (endsWith(absoluteFilePath, '.d.ts')) { + // TODO: Use sourceFile from the callback if + // https://github.com/Microsoft/TypeScript/issues/7438 + // is taken + const originalFile = absoluteFilePath.replace(this.tsOpts.outDir, this.tsOpts.rootDir) + .replace(/\.d\.ts$/, '.ts'); + const sourceFile = program.getSourceFile(originalFile); + this.emitMetadata(absoluteFilePath, sourceFile, typeChecker); + } }); if (this.genInternalTypings) { @@ -239,6 +258,22 @@ class DiffingTSCompiler implements DiffingBroccoliPlugin { } } + /** + * Emit a .metadata.json file to correspond to the .d.ts file if the module contains classes that + * use decorators or exported constants. + */ + private emitMetadata(dtsFileName: string, sourceFile: ts.SourceFile, + typeChecker: ts.TypeChecker) { + if (sourceFile) { + const metadata = this.metadataExtractor.getMetadata(sourceFile, typeChecker); + if (metadata && metadata.metadata) { + const metadataText = JSON.stringify(metadata); + const metadataFileName = dtsFileName.replace(/\.d.ts$/, '.metadata.json'); + fs.writeFileSync(metadataFileName, metadataText, FS_OPTS); + } + } + } + /** * There is a bug in TypeScript 1.6, where the sourceRoot and inlineSourceMap properties * are exclusive. This means that the sources property always contains relative paths diff --git a/tools/metadata/evaluator.ts b/tools/metadata/evaluator.ts new file mode 100644 index 0000000000..776b77b50a --- /dev/null +++ b/tools/metadata/evaluator.ts @@ -0,0 +1,304 @@ +import * as ts from 'typescript'; +import {Symbols} from './symbols'; + +function isMethodCallOf(callExpression: ts.CallExpression, memberName: string): boolean { + const expression = callExpression.expression; + if (expression.kind === ts.SyntaxKind.PropertyAccessExpression) { + const propertyAccessExpression = expression; + const name = propertyAccessExpression.name; + if (name.kind == ts.SyntaxKind.Identifier) { + return name.text === memberName; + } + } + return false; +} + +function isCallOf(callExpression: ts.CallExpression, ident: string): boolean { + const expression = callExpression.expression; + if (expression.kind === ts.SyntaxKind.Identifier) { + const identifier = expression; + return identifier.text === ident; + } + return false; +} + +/** + * ts.forEachChild stops iterating children when the callback return a truthy value. + * This method inverts this to implement an `every` style iterator. It will return + * true if every call to `cb` returns `true`. + */ +function everyNodeChild(node: ts.Node, cb: (node: ts.Node) => boolean) { + return !ts.forEachChild(node, node => !cb(node)); +} + +export interface SymbolReference { + __symbolic: string; // TODO: Change this to type "reference" when we move to TypeScript 1.8 + name: string; + module: string; +} + +function isPrimitive(value: any): boolean { + return Object(value) !== value; +} + +function isDefined(obj: any): boolean { + return obj !== undefined; +} + +/** + * Produce a symbolic representation of an expression folding values into their final value when + * possible. + */ +export class Evaluator { + constructor(private service: ts.LanguageService, private typeChecker: ts.TypeChecker, + private symbols: Symbols, private moduleNameOf: (fileName: string) => string) {} + + // TODO: Determine if the first declaration is deterministic. + private symbolFileName(symbol: ts.Symbol): string { + if (symbol) { + if (symbol.flags & ts.SymbolFlags.Alias) { + symbol = this.typeChecker.getAliasedSymbol(symbol); + } + const declarations = symbol.getDeclarations(); + if (declarations && declarations.length > 0) { + const sourceFile = declarations[0].getSourceFile(); + if (sourceFile) { + return sourceFile.fileName; + } + } + } + return undefined; + } + + private symbolReference(symbol: ts.Symbol): SymbolReference { + if (symbol) { + const name = symbol.name; + const module = this.moduleNameOf(this.symbolFileName(symbol)); + return {__symbolic: "reference", name, module}; + } + } + + private nodeSymbolReference(node: ts.Node): SymbolReference { + return this.symbolReference(this.typeChecker.getSymbolAtLocation(node)); + } + + nameOf(node: ts.Node): string { + if (node.kind == ts.SyntaxKind.Identifier) { + return (node).text; + } + return this.evaluateNode(node); + } + + /** + * Returns true if the expression represented by `node` can be folded into a literal expression. + * + * For example, a literal is always foldable. This means that literal expressions such as `1.2` + * `"Some value"` `true` `false` are foldable. + * + * - An object literal is foldable if all the properties in the literal are foldable. + * - An array literal is foldable if all the elements are foldable. + * - A call is foldable if it is a call to a Array.prototype.concat or a call to CONST_EXPR. + * - A property access is foldable if the object is foldable. + * - A array index is foldable if index expression is foldable and the array is foldable. + * - Binary operator expressions are foldable if the left and right expressions are foldable and + * it is one of '+', '-', '*', '/', '%', '||', and '&&'. + * - An identifier is foldable if a value can be found for its symbol is in the evaluator symbol + * table. + */ + public isFoldable(node: ts.Node) { + if (node) { + switch (node.kind) { + case ts.SyntaxKind.ObjectLiteralExpression: + return everyNodeChild(node, child => { + if (child.kind === ts.SyntaxKind.PropertyAssignment) { + const propertyAssignment = child; + return this.isFoldable(propertyAssignment.initializer) + } + return false; + }); + case ts.SyntaxKind.ArrayLiteralExpression: + return everyNodeChild(node, child => this.isFoldable(child)); + case ts.SyntaxKind.CallExpression: + const callExpression = node; + // We can fold a .concat(). + if (isMethodCallOf(callExpression, "concat") && callExpression.arguments.length === 1) { + const arrayNode = (callExpression.expression).expression; + if (this.isFoldable(arrayNode) && this.isFoldable(callExpression.arguments[0])) { + // It needs to be an array. + const arrayValue = this.evaluateNode(arrayNode); + if (arrayValue && Array.isArray(arrayValue)) { + return true; + } + } + } + // We can fold a call to CONST_EXPR + if (isCallOf(callExpression, "CONST_EXPR") && callExpression.arguments.length === 1) + return this.isFoldable(callExpression.arguments[0]); + return false; + case ts.SyntaxKind.NoSubstitutionTemplateLiteral: + case ts.SyntaxKind.StringLiteral: + case ts.SyntaxKind.NumericLiteral: + case ts.SyntaxKind.NullKeyword: + case ts.SyntaxKind.TrueKeyword: + case ts.SyntaxKind.FalseKeyword: + return true; + case ts.SyntaxKind.BinaryExpression: + const binaryExpression = node; + switch (binaryExpression.operatorToken.kind) { + case ts.SyntaxKind.PlusToken: + case ts.SyntaxKind.MinusToken: + case ts.SyntaxKind.AsteriskToken: + case ts.SyntaxKind.SlashToken: + case ts.SyntaxKind.PercentToken: + case ts.SyntaxKind.AmpersandAmpersandToken: + case ts.SyntaxKind.BarBarToken: + return this.isFoldable(binaryExpression.left) && + this.isFoldable(binaryExpression.right); + } + case ts.SyntaxKind.PropertyAccessExpression: + const propertyAccessExpression = node; + return this.isFoldable(propertyAccessExpression.expression); + case ts.SyntaxKind.ElementAccessExpression: + const elementAccessExpression = node; + return this.isFoldable(elementAccessExpression.expression) && + this.isFoldable(elementAccessExpression.argumentExpression); + case ts.SyntaxKind.Identifier: + const symbol = this.typeChecker.getSymbolAtLocation(node); + if (this.symbols.has(symbol)) return true; + break; + } + } + return false; + } + + /** + * Produce a JSON serialiable object representing `node`. The foldable values in the expression + * tree are folded. For example, a node representing `1 + 2` is folded into `3`. + */ + public evaluateNode(node: ts.Node): any { + switch (node.kind) { + case ts.SyntaxKind.ObjectLiteralExpression: + let obj = {}; + let allPropertiesDefined = true; + ts.forEachChild(node, child => { + switch (child.kind) { + case ts.SyntaxKind.PropertyAssignment: + const assignment = child; + const propertyName = this.nameOf(assignment.name); + const propertyValue = this.evaluateNode(assignment.initializer); + obj[propertyName] = propertyValue; + allPropertiesDefined = isDefined(propertyValue) && allPropertiesDefined; + } + }); + if (allPropertiesDefined) return obj; + break; + case ts.SyntaxKind.ArrayLiteralExpression: + let arr = []; + let allElementsDefined = true; + ts.forEachChild(node, child => { + const value = this.evaluateNode(child); + arr.push(value); + allElementsDefined = isDefined(value) && allElementsDefined; + }); + if (allElementsDefined) return arr; + break; + case ts.SyntaxKind.CallExpression: + const callExpression = node; + const args = callExpression.arguments.map(arg => this.evaluateNode(arg)); + if (this.isFoldable(callExpression)) { + if (isMethodCallOf(callExpression, "concat")) { + const arrayValue = this.evaluateNode( + (callExpression.expression).expression); + return arrayValue.concat(args[0]); + } + } + // Always fold a CONST_EXPR even if the argument is not foldable. + if (isCallOf(callExpression, "CONST_EXPR") && callExpression.arguments.length === 1) { + return args[0]; + } + const expression = this.evaluateNode(callExpression.expression); + if (isDefined(expression) && args.every(isDefined)) { + return { + __symbolic: "call", + expression: this.evaluateNode(callExpression.expression), + arguments: args + }; + } + break; + case ts.SyntaxKind.PropertyAccessExpression: { + const propertyAccessExpression = node; + const expression = this.evaluateNode(propertyAccessExpression.expression); + const member = this.nameOf(propertyAccessExpression.name); + if (this.isFoldable(propertyAccessExpression.expression)) return expression[member]; + if (isDefined(expression)) { + return {__symbolic: "select", expression, member}; + } + break; + } + case ts.SyntaxKind.ElementAccessExpression: { + const elementAccessExpression = node; + const expression = this.evaluateNode(elementAccessExpression.expression); + const index = this.evaluateNode(elementAccessExpression.argumentExpression); + if (this.isFoldable(elementAccessExpression.expression) && + this.isFoldable(elementAccessExpression.argumentExpression)) + return expression[index]; + if (isDefined(expression) && isDefined(index)) { + return { + __symbolic: "index", + expression, + index: this.evaluateNode(elementAccessExpression.argumentExpression) + }; + } + break; + } + case ts.SyntaxKind.Identifier: + const symbol = this.typeChecker.getSymbolAtLocation(node); + if (this.symbols.has(symbol)) return this.symbols.get(symbol); + return this.nodeSymbolReference(node); + case ts.SyntaxKind.NoSubstitutionTemplateLiteral: + return (node).text; + case ts.SyntaxKind.StringLiteral: + return (node).text; + case ts.SyntaxKind.NumericLiteral: + return parseFloat((node).text); + case ts.SyntaxKind.NullKeyword: + return null; + case ts.SyntaxKind.TrueKeyword: + return true; + case ts.SyntaxKind.FalseKeyword: + return false; + + case ts.SyntaxKind.BinaryExpression: + const binaryExpression = node; + const left = this.evaluateNode(binaryExpression.left); + const right = this.evaluateNode(binaryExpression.right); + if (isDefined(left) && isDefined(right)) { + if (isPrimitive(left) && isPrimitive(right)) + switch (binaryExpression.operatorToken.kind) { + case ts.SyntaxKind.PlusToken: + return left + right; + case ts.SyntaxKind.MinusToken: + return left - right; + case ts.SyntaxKind.AsteriskToken: + return left * right; + case ts.SyntaxKind.SlashToken: + return left / right; + case ts.SyntaxKind.PercentToken: + return left % right; + case ts.SyntaxKind.AmpersandAmpersandToken: + return left && right; + case ts.SyntaxKind.BarBarToken: + return left || right; + } + return { + __symbolic: "binop", + operator: binaryExpression.operatorToken.getText(), + left: left, + right: right + }; + } + break; + } + return undefined; + } +} diff --git a/tools/metadata/extractor.ts b/tools/metadata/extractor.ts new file mode 100644 index 0000000000..026f7722d4 --- /dev/null +++ b/tools/metadata/extractor.ts @@ -0,0 +1,92 @@ +import * as ts from 'typescript'; +import {Evaluator} from './evaluator'; +import {Symbols} from './symbols'; +import * as path from 'path'; + +const EXT_REGEX = /(\.ts|\.d\.ts|\.js|\.jsx|\.tsx)$/; +const NODE_MODULES = '/node_modules/'; +const NODE_MODULES_PREFIX = 'node_modules/'; + +function pathTo(from: string, to: string): string { + var result = path.relative(path.dirname(from), to); + if (path.dirname(result) === '.') { + result = '.' + path.sep + result; + } + return result; +} + +function moduleNameFromBaseName(moduleFileName: string, baseFileName: string): string { + // Remove the extension + moduleFileName = moduleFileName.replace(EXT_REGEX, ''); + + // Check for node_modules + const nodeModulesIndex = moduleFileName.lastIndexOf(NODE_MODULES); + if (nodeModulesIndex >= 0) { + return moduleFileName.substr(nodeModulesIndex + NODE_MODULES.length); + } + if (moduleFileName.lastIndexOf(NODE_MODULES_PREFIX, NODE_MODULES_PREFIX.length) !== -1) { + return moduleFileName.substr(NODE_MODULES_PREFIX.length); + } + + // Construct a simplified path from the file to the module + return pathTo(baseFileName, moduleFileName); +} + +// TODO: Support cross-module folding +export class MetadataExtractor { + constructor(private service: ts.LanguageService) {} + + /** + * Returns a JSON.stringify friendly form describing the decorators of the exported classes from + * the source file that is expected to correspond to a module. + */ + public getMetadata(sourceFile: ts.SourceFile, typeChecker: ts.TypeChecker): any { + const locals = new Symbols(); + const moduleNameOf = (fileName: string) => + moduleNameFromBaseName(fileName, sourceFile.fileName); + const evaluator = new Evaluator(this.service, typeChecker, locals, moduleNameOf); + + function objFromDecorator(decoratorNode: ts.Decorator): any { + return evaluator.evaluateNode(decoratorNode.expression); + } + + function classWithDecorators(classDeclaration: ts.ClassDeclaration): any { + return { + __symbolic: "class", + decorators: classDeclaration.decorators.map(decorator => objFromDecorator(decorator)) + }; + } + + let metadata: any; + const symbols = typeChecker.getSymbolsInScope(sourceFile, ts.SymbolFlags.ExportValue); + for (var symbol of symbols) { + for (var declaration of symbol.getDeclarations()) { + switch (declaration.kind) { + case ts.SyntaxKind.ClassDeclaration: + const classDeclaration = declaration; + if (classDeclaration.decorators) { + if (!metadata) metadata = {}; + metadata[classDeclaration.name.text] = classWithDecorators(classDeclaration) + } + break; + case ts.SyntaxKind.VariableDeclaration: + const variableDeclaration = declaration; + if (variableDeclaration.initializer) { + const value = evaluator.evaluateNode(variableDeclaration.initializer); + if (value !== undefined) { + if (evaluator.isFoldable(variableDeclaration.initializer)) { + // Record the value for use in other initializers + locals.set(symbol, value); + } + if (!metadata) metadata = {}; + metadata[evaluator.nameOf(variableDeclaration.name)] = + evaluator.evaluateNode(variableDeclaration.initializer); + } + } + break; + } + } + } + return metadata && {__symbolic: "module", module: moduleNameOf(sourceFile.fileName), metadata}; + } +} diff --git a/tools/metadata/symbols.ts b/tools/metadata/symbols.ts new file mode 100644 index 0000000000..9dfe2e2804 --- /dev/null +++ b/tools/metadata/symbols.ts @@ -0,0 +1,34 @@ +import * as ts from 'typescript'; + +// TOOD: Remove when tools directory is upgraded to support es6 target +interface Map { + has(v: V): boolean; + set(k: K, v: V): void; + get(k: K): V; +} +interface MapConstructor { + new(): Map; +} +declare var Map: MapConstructor; + +var a: Array; + +/** + * A symbol table of ts.Symbol to a folded value used during expression folding in Evaluator. + * + * This is a thin wrapper around a Map<> using the first declaration location instead of the symbol + * itself as the key. In the TypeScript binder and type checker, mulitple symbols are sometimes + * created for a symbol depending on what scope it is in (e.g. export vs. local). Using the + * declaration node as the key results in these duplicate symbols being treated as identical. + */ +export class Symbols { + private map = new Map(); + + public has(symbol: ts.Symbol): boolean { return this.map.has(symbol.declarations[0]); } + + public set(symbol: ts.Symbol, value): void { this.map.set(symbol.declarations[0], value); } + + public get(symbol: ts.Symbol): any { return this.map.get(symbol.declarations[0]); } + + static empty: Symbols = new Symbols(); +}