build(aio): refactor dgeni packages

This is to tidy up the `author-packagse`, which currently duplicates a
lot of the configuration in the main packages. We need to
DRY this up so that we don't fall foul of a change in one being missed in
the other.

aio/tools/transforms/angular-base-package/processors/checkUnbalancedBackTicks.js

@@ -0,0 +1,33 @@
+var _ = require('lodash');
+
+/**
+ * @dgProcessor checkUnbalancedBackTicks
+ * @description
+ * Searches the rendered content for an odd number of (```) backticks,
+ * which would indicate an unbalanced pair and potentially a typo in the
+ * source content.
+ */
+module.exports = function checkUnbalancedBackTicks(log, createDocMessage) {
+
+  var BACKTICK_REGEX = /^ *```/gm;
+
+  return {
+    // $runAfter: ['checkAnchorLinksProcessor'],
+    $runAfter: ['inlineTagProcessor'],
+    $runBefore: ['writeFilesProcessor'],
+    $process: function(docs) {
+      _.forEach(docs, function(doc) {
+        if (doc.renderedContent) {
+          var matches = doc.renderedContent.match(BACKTICK_REGEX);
+          if (matches && matches.length % 2 !== 0) {
+            doc.unbalancedBackTicks = true;
+            log.warn(createDocMessage(
+                'checkUnbalancedBackTicks processor: unbalanced backticks found in rendered content',
+                doc));
+            log.warn(doc.renderedContent);
+          }
+        }
+      });
+    }
+  };
+};

aio/tools/transforms/angular-base-package/processors/checkUnbalancedBackTicks.spec.js

@@ -0,0 +1,29 @@
+var testPackage = require('../../helpers/test-package');
+var Dgeni = require('dgeni');
+
+describe('checkUnbalancedBackTicks', function() {
+  var dgeni, injector, processor, log;
+
+  beforeEach(function() {
+    dgeni = new Dgeni([testPackage('angular-base-package')]);
+    injector = dgeni.configureInjector();
+    processor = injector.get('checkUnbalancedBackTicks');
+    log = injector.get('log');
+  });
+
+  it('should warn if there are an odd number of back ticks in the rendered content', function() {
+    var docs = [{
+      renderedContent: '```\n' +
+          'code block\n' +
+          '```\n' +
+          '```\n' +
+          'code block with missing closing back ticks\n'
+    }];
+
+    processor.$process(docs);
+
+    expect(log.warn).toHaveBeenCalledWith(
+        'checkUnbalancedBackTicks processor: unbalanced backticks found in rendered content - doc');
+    expect(docs[0].unbalancedBackTicks).toBe(true);
+  });
+});

aio/tools/transforms/angular-base-package/processors/convertToJson.js

@@ -0,0 +1,40 @@
+module.exports = function convertToJsonProcessor(log, createDocMessage) {
+
+  return {
+    $runAfter: ['checkUnbalancedBackTicks'],
+    $runBefore: ['writeFilesProcessor'],
+    docTypes: [],
+    $process: function(docs) {
+      const docTypes = this.docTypes;
+      docs.forEach((doc) => {
+        if (docTypes.indexOf(doc.docType) !== -1) {
+          let contents = doc.renderedContent || '';
+
+          let title = doc.title;
+
+          // We do allow an empty `title` but resort to `name` if it is not even defined
+          if (title === undefined) {
+            title = doc.name;
+          }
+
+          // If there is no title then try to extract it from the first h1 in the renderedContent
+          if (title === undefined) {
+            const match = /<h1[^>]*>(.+?)<\/h1>/.exec(contents);
+            if (match) {
+              title = match[1];
+            }
+          }
+
+          // If there is still no title then log a warning
+          if (title === undefined) {
+            title = '';
+            log.warn(createDocMessage('Title property expected', doc));
+          }
+
+          doc.renderedContent = JSON.stringify({ id: doc.path, title, contents }, null, 2);
+        }
+      });
+    }
+  };
+
+};

aio/tools/transforms/angular-base-package/processors/convertToJson.spec.js

@@ -0,0 +1,71 @@
+var testPackage = require('../../helpers/test-package');
+var Dgeni = require('dgeni');
+
+describe('convertToJson processor', () => {
+  var dgeni, injector, processor, log;
+
+  beforeAll(function() {
+    dgeni = new Dgeni([testPackage('angular-base-package')]);
+    injector = dgeni.configureInjector();
+    processor = injector.get('convertToJsonProcessor');
+    log = injector.get('log');
+    processor.docTypes = ['test-doc'];
+  });
+
+  it('should be part of the dgeni package', () => {
+    expect(processor).toBeDefined();
+  });
+
+  it('should convert the renderedContent to JSON', () => {
+    const docs = [{
+      docType: 'test-doc',
+      title: 'The Title',
+      name: 'The Name',
+      path: 'test/doc',
+      renderedContent: 'Some Content'
+    }];
+    processor.$process(docs);
+    expect(JSON.parse(docs[0].renderedContent).id).toEqual('test/doc');
+    expect(JSON.parse(docs[0].renderedContent).title).toEqual('The Title');
+    expect(JSON.parse(docs[0].renderedContent).contents).toEqual('Some Content');
+  });
+
+  it('should get the title from name if no title is specified', () => {
+    const docs = [{ docType: 'test-doc', name: 'The Name' }];
+    processor.$process(docs);
+    expect(JSON.parse(docs[0].renderedContent).title).toEqual('The Name');
+  });
+
+  it('should accept an empty title', () => {
+    const docs = [{ docType: 'test-doc', title: '' }];
+    processor.$process(docs);
+    expect(JSON.parse(docs[0].renderedContent).title).toEqual('');
+    expect(log.warn).not.toHaveBeenCalled();
+  });
+
+  it('should accept an empty name if title is not provided', () => {
+    const docs = [{ docType: 'test-doc', name: '' }];
+    processor.$process(docs);
+    expect(JSON.parse(docs[0].renderedContent).title).toEqual('');
+    expect(log.warn).not.toHaveBeenCalled();
+  });
+
+  it('should get the title from the first `h1` if no title nor name is specified', () => {
+    const docs = [{ docType: 'test-doc', renderedContent: '<div><h1 class="title">Some title</h1><article><h1>Article 1</h1></article></div>' }];
+    processor.$process(docs);
+    expect(JSON.parse(docs[0].renderedContent).contents).toEqual('<div><h1 class="title">Some title</h1><article><h1>Article 1</h1></article></div>');
+    expect(JSON.parse(docs[0].renderedContent).title).toEqual('Some title');
+  });
+
+  it('should set missing titles to empty', () => {
+    const docs = [{ docType: 'test-doc' }];
+    processor.$process(docs);
+    expect(JSON.parse(docs[0].renderedContent).title).toBe('');
+  });
+
+  it('should log a warning', () => {
+    const docs = [{ docType: 'test-doc' }];
+    processor.$process(docs);
+    expect(log.warn).toHaveBeenCalled();
+  });
+});

aio/tools/transforms/angular-base-package/processors/createOverviewDump.js

@@ -0,0 +1,24 @@
+var _ = require('lodash');
+
+module.exports = function createOverviewDump() {
+
+  return {
+    $runAfter: ['processing-docs'],
+    $runBefore: ['docs-processed'],
+    $process: function(docs) {
+      var overviewDoc = {
+        id: 'overview-dump',
+        aliases: ['overview-dump'],
+        path: 'overview-dump',
+        outputPath: 'overview-dump.html',
+        modules: []
+      };
+      _.forEach(docs, function(doc) {
+        if (doc.docType === 'module') {
+          overviewDoc.modules.push(doc);
+        }
+      });
+      docs.push(overviewDoc);
+    }
+  };
+};

aio/tools/transforms/angular-base-package/processors/fixInternalDocumentLinks.js

@@ -0,0 +1,24 @@
+/**
+ * @dgProcessor fixInternalDocumentLinks
+ * @description
+ * Add in the document path to links that start with a hash.
+ * This is important when the web app has a base href in place,
+ * since links like: `<a href="#some-id">` would get mapped to
+ * the URL `base/#some-id` even if the current location is `base/some/doc`.
+ */
+module.exports = function fixInternalDocumentLinks() {
+
+  var INTERNAL_LINK = /(<a [^>]*href=")(#[^"]*)/g;
+
+  return {
+    $runAfter: ['inlineTagProcessor'],
+    $runBefore: ['convertToJsonProcessor'],
+    $process: function(docs) {
+      docs.forEach(doc => {
+        doc.renderedContent = doc.renderedContent.replace(INTERNAL_LINK, (_, pre, hash) => {
+          return pre + doc.path + hash;
+        });
+      });
+    }
+  };
+};

aio/tools/transforms/angular-base-package/processors/fixInternalDocumentLinks.spec.js

@@ -0,0 +1,52 @@
+const testPackage = require('../../helpers/test-package');
+const processorFactory = require('./fixInternalDocumentLinks');
+const Dgeni = require('dgeni');
+
+describe('fixInternalDocumentLinks processor', () => {
+
+  it('should be available on the injector', () => {
+    const dgeni = new Dgeni([testPackage('angular-base-package')]);
+    const injector = dgeni.configureInjector();
+    const processor = injector.get('fixInternalDocumentLinks');
+    expect(processor.$process).toBeDefined();
+  });
+
+  it('should run before the correct processor', () => {
+    const processor = processorFactory();
+    expect(processor.$runBefore).toEqual(['convertToJsonProcessor']);
+  });
+
+  it('should run after the correct processor', () => {
+    const processor = processorFactory();
+    expect(processor.$runAfter).toEqual(['inlineTagProcessor']);
+  });
+
+  it('should prefix internal hash links with the current doc path', () => {
+    const processor = processorFactory();
+    const docs = [
+      {
+        path: 'some/doc',
+        renderedContent: `
+          <a href="http://google.com#q=angular">Google</a>
+          <a href="some/relative/path#some-id">Some Id</a>
+          <a href="#some-internal-id">Link to heading</a>
+          <a class="important" href="#some-internal-id">Link to heading</a>
+          <a href="#some-internal-id" target="_blank">Link to heading</a>
+        `
+      },
+    ];
+    processor.$process(docs);
+    expect(docs).toEqual([
+      {
+        path: 'some/doc',
+        renderedContent: `
+          <a href="http://google.com#q=angular">Google</a>
+          <a href="some/relative/path#some-id">Some Id</a>
+          <a href="some/doc#some-internal-id">Link to heading</a>
+          <a class="important" href="some/doc#some-internal-id">Link to heading</a>
+          <a href="some/doc#some-internal-id" target="_blank">Link to heading</a>
+        `
+      },
+    ]);
+  });
+});

aio/tools/transforms/angular-base-package/processors/generateKeywords.js

@@ -0,0 +1,142 @@
+'use strict';
+
+var fs = require('fs');
+var path = require('canonical-path');
+
+/**
+ * @dgProcessor generateKeywordsProcessor
+ * @description
+ * This processor extracts all the keywords from each document and creates
+ * a new document that will be rendered as a JavaScript file containing all
+ * this data.
+ */
+module.exports = function generateKeywordsProcessor(log, readFilesProcessor) {
+  return {
+    ignoreWordsFile: undefined,
+    propertiesToIgnore: [],
+    docTypesToIgnore: [],
+    outputFolder: '',
+    $validate: {
+      ignoreWordsFile: {},
+      docTypesToIgnore: {},
+      propertiesToIgnore: {},
+      outputFolder: {presence: true}
+    },
+    $runAfter: ['paths-computed'],
+    $runBefore: ['rendering-docs'],
+    $process: function(docs) {
+
+      // Keywords to ignore
+      var wordsToIgnore = [];
+      var propertiesToIgnore;
+      var docTypesToIgnore;
+
+      // Keywords start with "ng:" or one of $, _ or a letter
+      var KEYWORD_REGEX = /^((ng:|[$_a-z])[\w\-_]+)/;
+
+      // Load up the keywords to ignore, if specified in the config
+      if (this.ignoreWordsFile) {
+        var ignoreWordsPath = path.resolve(readFilesProcessor.basePath, this.ignoreWordsFile);
+        wordsToIgnore = fs.readFileSync(ignoreWordsPath, 'utf8').toString().split(/[,\s\n\r]+/gm);
+
+        log.debug('Loaded ignore words from "' + ignoreWordsPath + '"');
+        log.silly(wordsToIgnore);
+      }
+
+      propertiesToIgnore = convertToMap(this.propertiesToIgnore);
+      log.debug('Properties to ignore', propertiesToIgnore);
+      docTypesToIgnore = convertToMap(this.docTypesToIgnore);
+      log.debug('Doc types to ignore', docTypesToIgnore);
+
+      var ignoreWordsMap = convertToMap(wordsToIgnore);
+
+      // If the title contains a name starting with ng, e.g. "ngController", then add the module
+      // name
+      // without the ng to the title text, e.g. "controller".
+      function extractTitleWords(title) {
+        var match = /ng([A-Z]\w*)/.exec(title);
+        if (match) {
+          title = title + ' ' + match[1].toLowerCase();
+        }
+        return title;
+      }
+
+      function extractWords(text, words, keywordMap) {
+        var tokens = text.toLowerCase().split(/[.\s,`'"#]+/mg);
+        tokens.forEach(function(token) {
+          var match = token.match(KEYWORD_REGEX);
+          if (match) {
+            var key = match[1];
+            if (!keywordMap[key]) {
+              keywordMap[key] = true;
+              words.push(key);
+            }
+          }
+        });
+      }
+
+
+      const filteredDocs = docs
+          // We are not interested in some docTypes
+          .filter(function(doc) { return !docTypesToIgnore[doc.docType]; })
+          // Ignore internals and private exports (indicated by the ɵ prefix)
+          .filter(function(doc) { return !doc.internal && !doc.privateExport; });
+
+      filteredDocs.forEach(function(doc) {
+
+
+        var words = [];
+        var keywordMap = Object.assign({}, ignoreWordsMap);
+        var members = [];
+        var membersMap = {};
+
+        // Search each top level property of the document for search terms
+        Object.keys(doc).forEach(function(key) {
+          const value = doc[key];
+
+          if (isString(value) && !propertiesToIgnore[key]) {
+            extractWords(value, words, keywordMap);
+          }
+
+          if (key === 'methods' || key === 'properties' || key === 'events') {
+            value.forEach(function(member) { extractWords(member.name, members, membersMap); });
+          }
+        });
+
+
+        doc.searchTerms = {
+          titleWords: extractTitleWords(doc.title || doc.name),
+          keywords: words.sort().join(' '),
+          members: members.sort().join(' ')
+        };
+
+      });
+
+      var searchData =
+          filteredDocs.filter(function(page) { return page.searchTerms; }).map(function(page) {
+            return Object.assign(
+                {path: page.path, title: page.name || page.title, type: page.docType}, page.searchTerms);
+          });
+
+      docs.push({
+        docType: 'json-doc',
+        id: 'search-data-json',
+        template: 'json-doc.template.json',
+        path: this.outputFolder + '/search-data.json',
+        outputPath: this.outputFolder + '/search-data.json',
+        data: searchData
+      });
+    }
+  };
+};
+
+
+function isString(value) {
+  return typeof value == 'string';
+}
+
+function convertToMap(collection) {
+  const obj = {};
+  collection.forEach(key => { obj[key] = true; });
+  return obj;
+}

aio/tools/transforms/angular-base-package/processors/generateKeywords.spec.js

@@ -0,0 +1,41 @@
+const testPackage = require('../../helpers/test-package');
+const mockLogger = require('dgeni/lib/mocks/log')(false);
+const processorFactory = require('./generateKeywords');
+const Dgeni = require('dgeni');
+
+const mockReadFilesProcessor = {
+  basePath: 'base/path'
+};
+
+describe('generateKeywords processor', () => {
+
+  it('should be available on the injector', () => {
+    const dgeni = new Dgeni([testPackage('angular-base-package')]);
+    const injector = dgeni.configureInjector();
+    const processor = injector.get('generateKeywordsProcessor');
+    expect(processor.$process).toBeDefined();
+  });
+
+  it('should run after the correct processor', () => {
+    const processor = processorFactory(mockLogger, mockReadFilesProcessor);
+    expect(processor.$runAfter).toEqual(['paths-computed']);
+  });
+
+  it('should run before the correct processor', () => {
+    const processor = processorFactory(mockLogger, mockReadFilesProcessor);
+    expect(processor.$runBefore).toEqual(['rendering-docs']);
+  });
+
+  it('should ignore internal and private exports', () => {
+    const processor = processorFactory(mockLogger, mockReadFilesProcessor);
+    const docs = [
+      { docType: 'class', name: 'PublicExport' },
+      { docType: 'class', name: 'PrivateExport', privateExport: true },
+      { docType: 'class', name: 'InternalExport', internal: true }
+    ];
+    processor.$process(docs);
+    expect(docs[docs.length - 1].data).toEqual([
+      jasmine.objectContaining({ title: 'PublicExport', type: 'class'})
+    ]);
+  });
+});