From db97d73c3bbb7cb05f7cf039761efecd4592c90a Mon Sep 17 00:00:00 2001 From: Martin Probst Date: Mon, 13 Apr 2015 16:22:23 -0700 Subject: [PATCH] feat(build): Move HTML copying into the broccoli task. This includes all tasks to construct a Dart tree, except for formatting, and reverse engineers/refactors the various copy tools for added more sanity. --- gulpfile.js | 9 +- tools/broccoli/broccoli-filter.d.ts | 5 +- tools/broccoli/trees/dart_tree.ts | 127 +++++++++++++++++++++++++--- tools/broccoli/trees/multi_copy.ts | 47 ++++++++++ 4 files changed, 167 insertions(+), 21 deletions(-) create mode 100644 tools/broccoli/trees/multi_copy.ts diff --git a/gulpfile.js b/gulpfile.js index dfaed7518a..17b068e1ac 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -294,7 +294,7 @@ gulp.task('build/clean.docs', clean(gulp, gulpPlugins, { // ------------ // transpile -gulp.task('build/transpile.dart', ['build.broccoli.tools'], function() { +gulp.task('build/tree.dart', ['build.broccoli.tools'], function() { return getBroccoli().forDartTree().buildOnce(); }); @@ -617,12 +617,7 @@ gulp.task('test.transpiler.unittest', function() { // Builds all Dart packages, but does not compile them gulp.task('build/packages.dart', function(done) { - runSequence( - 'build/transpile.dart', // Creates the folder structure needed by subsequent tasks. - ['build/html.dart', 'build/copy.dart', 'build/multicopy.dart'], - 'build/format.dart', - done - ); + runSequence('build/tree.dart', 'build/format.dart', done); }); // Builds and compiles all Dart packages diff --git a/tools/broccoli/broccoli-filter.d.ts b/tools/broccoli/broccoli-filter.d.ts index 6944d24f58..3a04ebea2a 100644 --- a/tools/broccoli/broccoli-filter.d.ts +++ b/tools/broccoli/broccoli-filter.d.ts @@ -1,8 +1,11 @@ /// +interface FilterOptions { + extensions?: string[] +} declare class Filter { - constructor(inputTree: any); + constructor(inputTree: any, options?: FilterOptions); processString(contents: string, relativePath: string): string; // NB: This function is probably not intended as part of the public API processFile(srcDir: string, destDir: string, relativePath: string): Promise; diff --git a/tools/broccoli/trees/dart_tree.ts b/tools/broccoli/trees/dart_tree.ts index ed69b2d049..8a3077fb79 100644 --- a/tools/broccoli/trees/dart_tree.ts +++ b/tools/broccoli/trees/dart_tree.ts @@ -1,32 +1,65 @@ +/// 'use strict'; +import {MultiCopy} from './multi_copy'; var Funnel = require('broccoli-funnel'); +var glob = require('glob'); +var mergeTrees = require('broccoli-merge-trees'); var path = require('path'); +var renderLodashTemplate = require('broccoli-lodash'); var replace = require('broccoli-replace'); var stew = require('broccoli-stew'); var ts2dart = require('../broccoli-ts2dart'); -module.exports = function makeDartTree() { - // Transpile everything in 'modules'... - var modulesTree = new Funnel('modules', { - include: ['**/*.js', '**/*.ts', '**/*.dart'], // .dart file available means don't translate. - exclude: ['rtts_assert/**/*'], // ... except for the rtts_asserts (don't apply to Dart). - destDir: '/' // Remove the 'modules' prefix. - }); +/** + * A funnel starting at modules, including the given filters, and moving into the root. + * @param include Include glob filters. + */ +function modulesFunnel(include: string[], exclude?: string[]) { + return new Funnel('modules', {include, destDir: '/', exclude}); +} - // Transpile to dart. - var dartTree = ts2dart.transpile(modulesTree); +/** + * Replaces $SCRIPT$ in .html files with actual \n'; + } + var scriptName = relativePath.replace(/.*\/([^/]+)\.html$/, '$1.dart'); + scriptTags += '\n' + + ''; + return content.replace('$SCRIPTS$', scriptTags); +} +function stripModulePrefix(relativePath: string): string { + if (!relativePath.match(/^modules\//)) { + throw new Error('Expected path to root at modules/: ' + relativePath); + } + return relativePath.replace(/^modules\//, ''); +} + +function getSourceTree() { + // Transpile everything in 'modules' except for rtts_assertions. + var tsInputTree = modulesFunnel(['**/*.js', '**/*.ts', '**/*.dart'], ['rtts_assert/**/*']); + var transpiled = ts2dart.transpile(tsInputTree); + // Native sources, dart only examples, etc. + var dartSrcs = modulesFunnel(['**/*.dart']); + return mergeTrees([transpiled, dartSrcs]); +} + +function fixDartFolderLayout(sourceTree) { // Move around files to match Dart's layout expectations. - dartTree = stew.rename(dartTree, function(relativePath) { + return stew.rename(sourceTree, function(relativePath) { // If a file matches the `pattern`, insert the given `insertion` as the second path part. var replacements = [ {pattern: /^benchmarks\/test\//, insertion: ''}, {pattern: /^benchmarks\//, insertion: 'web'}, {pattern: /^benchmarks_external\/test\//, insertion: ''}, {pattern: /^benchmarks_external\//, insertion: 'web'}, - {pattern: /^example.?\//, insertion: 'web/'}, - {pattern: /^example.?\/test\//, insertion: ''}, + {pattern: /^examples\/test\//, insertion: ''}, + {pattern: /^examples\//, insertion: 'web/'}, {pattern: /^[^\/]*\/test\//, insertion: ''}, {pattern: /^./, insertion: 'lib'}, // catch all. ]; @@ -41,7 +74,75 @@ module.exports = function makeDartTree() { } throw new Error('Failed to match any path: ' + relativePath); }); +} + +function getHtmlSourcesTree() { + // Replace $SCRIPT$ markers in HTML files. + var htmlSrcsTree = stew.map(modulesFunnel(['*/src/**/*.html']), replaceScriptTagInHtml); + // Copy a url_params_to_form.js for each benchmark html file. + var urlParamsToFormTree = new MultiCopy('', { + srcPath: 'tools/build/snippets/url_params_to_form.js', + targetPatterns: ['modules/benchmarks*/src/*', 'modules/benchmarks*/src/*/*'], + }); + urlParamsToFormTree = stew.rename(urlParamsToFormTree, stripModulePrefix); + return mergeTrees([htmlSrcsTree, urlParamsToFormTree]); +} + + +function getTemplatedPubspecsTree() { + // The JSON structure for templating pubspec.yaml files. + var BASE_PACKAGE_JSON = require('../../../package.json'); + var COMMON_PACKAGE_JSON = { + version: BASE_PACKAGE_JSON.version, + homepage: BASE_PACKAGE_JSON.homepage, + bugs: BASE_PACKAGE_JSON.bugs, + license: BASE_PACKAGE_JSON.license, + contributors: BASE_PACKAGE_JSON.contributors, + dependencies: BASE_PACKAGE_JSON.dependencies, + devDependencies: { + "yargs": BASE_PACKAGE_JSON.devDependencies['yargs'], + "gulp-sourcemaps": BASE_PACKAGE_JSON.devDependencies['gulp-sourcemaps'], + "gulp-traceur": BASE_PACKAGE_JSON.devDependencies['gulp-traceur'], + "gulp": BASE_PACKAGE_JSON.devDependencies['gulp'], + "gulp-rename": BASE_PACKAGE_JSON.devDependencies['gulp-rename'], + "through2": BASE_PACKAGE_JSON.devDependencies['through2'] + } + }; + // Generate pubspec.yaml from templates. + // Lodash insists on dropping one level of extension, so first artificially rename the yaml + // files to .yaml.template. + var pubspecs = stew.rename(modulesFunnel(['**/pubspec.yaml']), '.yaml', '.yaml.template'); + // Then render the templates. + return renderLodashTemplate( + pubspecs, + {files: ['**/pubspec.yaml.template'], context: {'packageJson': COMMON_PACKAGE_JSON}}); +} + +function getDocsTree() { + // LICENSE files + var licenses = new MultiCopy('', { + srcPath: 'LICENSE', + targetPatterns: ['modules/*'], + exclude: ['*/rtts_assert'], // Not in dart. + }); + licenses = stew.rename(licenses, stripModulePrefix); + + // Documentation. + // Rename *.dart.md -> *.dart. + var mdTree = stew.rename(modulesFunnel(['**/*.dart.md']), + relativePath => relativePath.replace(/\.dart\.md$/, '.md')); + // Copy all assets, ignore .js. and .dart. (handled above). + var docs = modulesFunnel(['**/*.md', '**/*.png', '**/*.html', '**/*.css'], + ['**/*.js.md', '**/*.dart.md']); + return mergeTrees([licenses, mdTree, docs]); +} + +module.exports = function makeDartTree() { + var sourceTree = mergeTrees([getSourceTree(), getHtmlSourcesTree()]); + sourceTree = fixDartFolderLayout(sourceTree); + + var mergedResult = mergeTrees([sourceTree, getTemplatedPubspecsTree(), getDocsTree()]); // Move the tree under the 'dart' folder. - return stew.mv(dartTree, 'dart'); + return stew.mv(mergedResult, 'dart'); }; diff --git a/tools/broccoli/trees/multi_copy.ts b/tools/broccoli/trees/multi_copy.ts new file mode 100644 index 0000000000..e9d9cf272b --- /dev/null +++ b/tools/broccoli/trees/multi_copy.ts @@ -0,0 +1,47 @@ +/// +/// +import Writer = require('broccoli-writer'); +import fs = require('fs'); +import fsx = require('fs-extra'); +var minimatch = require('minimatch'); +var path = require('path'); +var glob = require('glob'); + +export interface MultiCopyOptions { + /** The path of the file to copy. */ + srcPath: string, + /** A list of glob patterns of folders to copy to, matched against the input tree. */ + targetPatterns: string[], + /** List of glob patterns to *not* copy to, matched against the matches from `targetPatterns`. */ + exclude?: string[], +} + +/** + * A writer that copies an input file from an input path into (potentially many) output locations + * given by glob patterns, . + */ +export class MultiCopy extends Writer { + constructor(private inputTree, private options: MultiCopyOptions) { super(); } + + write(readTree: (tree) => Promise, destDir: string): Promise { + return readTree(this.inputTree) + .then((inputPath: string) => { + var fileName = path.basename(this.options.srcPath); + var data = fs.readFileSync(path.join(inputPath, this.options.srcPath), 'utf-8'); + + this.options.targetPatterns.forEach(pattern => { + var paths: string[] = glob.sync(pattern); + paths = paths.filter(p => fs.statSync(p).isDirectory()); + if (this.options.exclude) { + paths = paths.filter(p => !this.options.exclude.some((excl) => minimatch(p, excl))); + } + paths.forEach(p => { + var folder = path.join(destDir, p); + fsx.mkdirsSync(folder); + var outputPath = path.join(folder, fileName); + fs.writeFileSync(outputPath, data); + }); + }); + }); + } +}