ci(docs-infra): rename 'upload-server' to 'preview-server'

The server no longer has files uploaded to it. Instead it is more
accurate to refer to it as dealing with "previews" of PRs.

aio/aio-builds-setup/dockerbuild/scripts-js/lib/preview-server/build-creator.ts

@@ -0,0 +1,145 @@
+// Imports
+import * as cp from 'child_process';
+import {EventEmitter} from 'events';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as shell from 'shelljs';
+import {HIDDEN_DIR_PREFIX} from '../common/constants';
+import {assertNotMissingOrEmpty, computeShortSha, createLogger} from '../common/utils';
+import {ChangedPrVisibilityEvent, CreatedBuildEvent} from './build-events';
+import {PreviewServerError} from './preview-error';
+
+// Classes
+export class BuildCreator extends EventEmitter {
+
+  private logger = createLogger('BuildCreator');
+
+  // Constructor
+  constructor(protected buildsDir: string) {
+    super();
+    assertNotMissingOrEmpty('buildsDir', buildsDir);
+  }
+
+  // Methods - Public
+  public create(pr: number, sha: string, archivePath: string, isPublic: boolean): Promise<void> {
+    // Use only part of the SHA for more readable URLs.
+    sha = computeShortSha(sha);
+
+    const {newPrDir: prDir} = this.getCandidatePrDirs(pr, isPublic);
+    const shaDir = path.join(prDir, sha);
+    let dirToRemoveOnError: string;
+
+    return Promise.resolve().
+      // If the same PR exists with different visibility, update the visibility first.
+      then(() => this.updatePrVisibility(pr, isPublic)).
+      then(() => Promise.all([this.exists(prDir), this.exists(shaDir)])).
+      then(([prDirExisted, shaDirExisted]) => {
+        if (shaDirExisted) {
+          const publicOrNot = isPublic ? 'public' : 'non-public';
+          throw new PreviewServerError(409, `Request to overwrite existing ${publicOrNot} directory: ${shaDir}`);
+        }
+
+        dirToRemoveOnError = prDirExisted ? shaDir : prDir;
+
+        return Promise.resolve().
+          then(() => shell.mkdir('-p', shaDir)).
+          then(() => this.extractArchive(archivePath, shaDir)).
+          then(() => this.emit(CreatedBuildEvent.type, new CreatedBuildEvent(+pr, sha, isPublic))).
+          then(() => undefined);
+      }).
+      catch(err => {
+        if (dirToRemoveOnError) {
+          shell.rm('-rf', dirToRemoveOnError);
+        }
+
+        if (!(err instanceof PreviewServerError)) {
+          err = new PreviewServerError(500, `Error while creating preview at: ${shaDir}\n${err}`);
+        }
+
+        throw err;
+      });
+  }
+
+  public updatePrVisibility(pr: number, makePublic: boolean): Promise<boolean> {
+    const {oldPrDir: otherVisPrDir, newPrDir: targetVisPrDir} = this.getCandidatePrDirs(pr, makePublic);
+
+    return Promise.
+      all([this.exists(otherVisPrDir), this.exists(targetVisPrDir)]).
+      then(([otherVisPrDirExisted, targetVisPrDirExisted]) => {
+        if (!otherVisPrDirExisted) {
+          // No visibility change: Either the visibility is up-to-date or the PR does not exist.
+          return false;
+        } else if (targetVisPrDirExisted) {
+          // Error: Directories for both visibilities exist.
+          throw new PreviewServerError(409,
+              `Request to move '${otherVisPrDir}' to existing directory '${targetVisPrDir}'.`);
+        }
+
+        // Visibility change: Moving `otherVisPrDir` to `targetVisPrDir`.
+        return Promise.resolve().
+          then(() => shell.mv(otherVisPrDir, targetVisPrDir)).
+          then(() => this.listShasByDate(targetVisPrDir)).
+          then(shas => this.emit(ChangedPrVisibilityEvent.type, new ChangedPrVisibilityEvent(+pr, shas, makePublic))).
+          then(() => true);
+      }).
+      catch(err => {
+        if (!(err instanceof PreviewServerError)) {
+          err = new PreviewServerError(500, `Error while making PR ${pr} ${makePublic ? 'public' : 'hidden'}.\n${err}`);
+        }
+
+        throw err;
+      });
+  }
+
+  // Methods - Protected
+  protected exists(fileOrDir: string): Promise<boolean> {
+    return new Promise(resolve => fs.access(fileOrDir, err => resolve(!err)));
+  }
+
+  protected extractArchive(inputFile: string, outputDir: string): Promise<void> {
+    return new Promise<void>((resolve, reject) => {
+      const cmd = `tar --extract --gzip --directory "${outputDir}" --file "${inputFile}"`;
+
+      cp.exec(cmd, (err, _stdout, stderr) => {
+        if (err) {
+          return reject(err);
+        }
+
+        if (stderr) {
+          this.logger.warn(stderr);
+        }
+
+        try {
+          // Undocumented signature (see https://github.com/shelljs/shelljs/pull/663).
+          (shell as any).chmod('-R', 'a-w', outputDir);
+          shell.rm('-f', inputFile);
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      });
+    });
+  }
+
+  protected getCandidatePrDirs(pr: number, isPublic: boolean): {oldPrDir: string, newPrDir: string} {
+    const hiddenPrDir = path.join(this.buildsDir, HIDDEN_DIR_PREFIX + pr);
+    const publicPrDir = path.join(this.buildsDir, `${pr}`);
+
+    const oldPrDir = isPublic ? hiddenPrDir : publicPrDir;
+    const newPrDir = isPublic ? publicPrDir : hiddenPrDir;
+
+    return {oldPrDir, newPrDir};
+  }
+
+  protected listShasByDate(inputDir: string): Promise<string[]> {
+    return Promise.resolve().
+      then(() => shell.ls('-l', inputDir) as any as Promise<(fs.Stats & {name: string})[]>).
+      // Keep directories only.
+      // (Also, convert to standard Array - ShellJS provides custom `sort()` method for sorting file contents.)
+      then(items => items.filter(item => item.isDirectory())).
+      // Sort by modification date.
+      then(items => items.sort((a, b) => a.mtime.getTime() - b.mtime.getTime())).
+      // Return directory names.
+      then(items => items.map(item => item.name));
+  }
+}

aio/aio-builds-setup/dockerbuild/scripts-js/lib/preview-server/build-events.ts

@@ -0,0 +1,16 @@
+// Classes
+export class ChangedPrVisibilityEvent {
+  // Properties - Public, Static
+  public static type = 'pr.changedVisibility';
+
+  // Constructor
+  constructor(public pr: number, public shas: string[], public isPublic: boolean) {}
+}
+
+export class CreatedBuildEvent {
+  // Properties - Public, Static
+  public static type = 'build.created';
+
+  // Constructor
+  constructor(public pr: number, public sha: string, public isPublic: boolean) {}
+}

aio/aio-builds-setup/dockerbuild/scripts-js/lib/preview-server/build-retriever.ts

@@ -0,0 +1,83 @@
+import * as fs from 'fs';
+import fetch from 'node-fetch';
+import {dirname} from 'path';
+import {mkdir} from 'shelljs';
+import {promisify} from 'util';
+import {CircleCiApi} from '../common/circle-ci-api';
+import {assert, assertNotMissingOrEmpty, computeArtifactDownloadPath, createLogger} from '../common/utils';
+import {PreviewServerError} from './preview-error';
+
+export interface GithubInfo {
+  org: string;
+  pr: number;
+  repo: string;
+  sha: string;
+  success: boolean;
+}
+
+/**
+ * A helper that can get information about builds and download build artifacts.
+ */
+export class BuildRetriever {
+  private logger = createLogger('BuildRetriever');
+  constructor(private api: CircleCiApi, private downloadSizeLimit: number, private downloadDir: string) {
+    assert(downloadSizeLimit > 0, 'Invalid parameter "downloadSizeLimit" should be a number greater than 0.');
+    assertNotMissingOrEmpty('downloadDir', downloadDir);
+  }
+
+  /**
+   * Get GitHub information about a build
+   * @param buildNum The number of the build for which to retrieve the info.
+   * @returns The Github org, repo, PR and latest SHA for the specified build.
+   */
+  public async getGithubInfo(buildNum: number): Promise<GithubInfo> {
+    const buildInfo = await this.api.getBuildInfo(buildNum);
+    const githubInfo: GithubInfo = {
+      org: buildInfo.username,
+      pr: getPrfromBranch(buildInfo.branch),
+      repo: buildInfo.reponame,
+      sha: buildInfo.vcs_revision,
+      success: !buildInfo.failed,
+    };
+    return githubInfo;
+  }
+
+  /**
+   * Make a request to the given URL for a build artifact and store it locally.
+   * @param buildNum the number of the CircleCI build whose artifact we want to download.
+   * @param pr the number of the PR that triggered the CircleCI build.
+   * @param sha the commit in the PR that triggered the CircleCI build.
+   * @param artifactPath the path on CircleCI where the artifact was stored.
+   * @returns A promise to the file path where the downloaded file was stored.
+   */
+  public async downloadBuildArtifact(buildNum: number, pr: number, sha: string, artifactPath: string): Promise<string> {
+    try {
+      const outPath = computeArtifactDownloadPath(this.downloadDir, pr, sha, artifactPath);
+      const downloadExists = await new Promise(resolve => fs.exists(outPath, exists => resolve(exists)));
+      if (!downloadExists) {
+        const url = await this.api.getBuildArtifactUrl(buildNum, artifactPath);
+        const response = await fetch(url, {size: this.downloadSizeLimit});
+        if (response.status !== 200) {
+          throw new PreviewServerError(response.status, `Error ${response.status} - ${response.statusText}`);
+        }
+        const buffer = await response.buffer();
+        mkdir('-p', dirname(outPath));
+        await promisify(fs.writeFile)(outPath, buffer);
+      }
+      return outPath;
+    } catch (error) {
+      this.logger.warn(error);
+      const status = (error.type === 'max-size') ? 413 : 500;
+      throw new PreviewServerError(status, `CircleCI artifact download failed (${error.message || error})`);
+    }
+  }
+}
+
+function getPrfromBranch(branch: string): number {
+  // CircleCI only exposes PR numbers via the `branch` field :-(
+  const match = /^pull\/(\d+)$/.exec(branch);
+  if (!match) {
+    throw new Error(`No PR found in branch field: ${branch}`);
+  }
+  return +match[1];
+}

aio/aio-builds-setup/dockerbuild/scripts-js/lib/preview-server/build-verifier.ts

@@ -0,0 +1,46 @@
+import {GithubPullRequests, PullRequest} from '../common/github-pull-requests';
+import {GithubTeams} from '../common/github-teams';
+import {assertNotMissingOrEmpty} from '../common/utils';
+
+/**
+ * A helper to verify whether builds are trusted.
+ */
+export class BuildVerifier {
+  /**
+   * Construct a new BuildVerifier instance.
+   * @param prs A helper to access PR information.
+   * @param teams A helper to access Github team information.
+   * @param allowedTeamSlugs The teams that are trusted.
+   * @param trustedPrLabel The github label that indicates that a PR is trusted.
+   */
+  constructor(protected prs: GithubPullRequests, protected teams: GithubTeams,
+              protected allowedTeamSlugs: string[], protected trustedPrLabel: string) {
+    assertNotMissingOrEmpty('allowedTeamSlugs', allowedTeamSlugs && allowedTeamSlugs.join(''));
+    assertNotMissingOrEmpty('trustedPrLabel', trustedPrLabel);
+  }
+
+  /**
+   * Check whether a PR contains files that are significant to the build.
+   * @param pr The number of the PR to check
+   * @param significantFilePattern A regex that selects files that are significant.
+   */
+  public async getSignificantFilesChanged(pr: number, significantFilePattern: RegExp): Promise<boolean> {
+    const files = await this.prs.fetchFiles(pr);
+    return files.some(file => significantFilePattern.test(file.filename));
+  }
+
+  /**
+   * Check whether a PR is trusted.
+   * @param pr The number of the PR to check.
+   * @returns true if the PR is trusted.
+   */
+  public async getPrIsTrusted(pr: number): Promise<boolean> {
+    const prInfo = await this.prs.fetch(pr);
+    return this.hasLabel(prInfo, this.trustedPrLabel) ||
+           (await this.teams.isMemberBySlug(prInfo.user.login, this.allowedTeamSlugs));
+  }
+
+  protected hasLabel(prInfo: PullRequest, label: string): boolean {
+    return prInfo.labels.some(labelObj => labelObj.name === label);
+  }
+}

aio/aio-builds-setup/dockerbuild/scripts-js/lib/preview-server/index.ts

@@ -0,0 +1,41 @@
+// Imports
+import {AIO_DOWNLOADS_DIR} from '../common/constants';
+import {
+  AIO_ARTIFACT_MAX_SIZE,
+  AIO_ARTIFACT_PATH,
+  AIO_BUILDS_DIR,
+  AIO_CIRCLE_CI_TOKEN,
+  AIO_DOMAIN_NAME,
+  AIO_GITHUB_ORGANIZATION,
+  AIO_GITHUB_REPO,
+  AIO_GITHUB_TEAM_SLUGS,
+  AIO_GITHUB_TOKEN,
+  AIO_PREVIEW_SERVER_HOSTNAME,
+  AIO_PREVIEW_SERVER_PORT,
+  AIO_SIGNIFICANT_FILES_PATTERN,
+  AIO_TRUSTED_PR_LABEL,
+} from '../common/env-variables';
+import {PreviewServerFactory} from './preview-server-factory';
+
+// Run
+_main();
+
+// Functions
+function _main(): void {
+  PreviewServerFactory
+    .create({
+      buildArtifactPath: AIO_ARTIFACT_PATH,
+      buildsDir: AIO_BUILDS_DIR,
+      circleCiToken: AIO_CIRCLE_CI_TOKEN,
+      domainName: AIO_DOMAIN_NAME,
+      downloadSizeLimit: AIO_ARTIFACT_MAX_SIZE,
+      downloadsDir: AIO_DOWNLOADS_DIR,
+      githubOrg: AIO_GITHUB_ORGANIZATION,
+      githubRepo: AIO_GITHUB_REPO,
+      githubTeamSlugs: AIO_GITHUB_TEAM_SLUGS.split(','),
+      githubToken: AIO_GITHUB_TOKEN,
+      significantFilesPattern: AIO_SIGNIFICANT_FILES_PATTERN,
+      trustedPrLabel: AIO_TRUSTED_PR_LABEL,
+    })
+    .listen(AIO_PREVIEW_SERVER_PORT, AIO_PREVIEW_SERVER_HOSTNAME);
+}

aio/aio-builds-setup/dockerbuild/scripts-js/lib/preview-server/preview-error.ts

@@ -0,0 +1,8 @@
+// Classes
+export class PreviewServerError extends Error {
+  // Constructor
+  constructor(public status: number = 500, message?: string) {
+    super(message);
+    Object.setPrototypeOf(this, PreviewServerError.prototype);
+  }
+}

aio/aio-builds-setup/dockerbuild/scripts-js/lib/preview-server/preview-server-factory.ts

@@ -0,0 +1,183 @@
+// Imports
+import * as bodyParser from 'body-parser';
+import * as express from 'express';
+import * as http from 'http';
+import {CircleCiApi} from '../common/circle-ci-api';
+import {GithubApi} from '../common/github-api';
+import {GithubPullRequests} from '../common/github-pull-requests';
+import {GithubTeams} from '../common/github-teams';
+import {assert, assertNotMissingOrEmpty, createLogger} from '../common/utils';
+import {BuildCreator} from './build-creator';
+import {ChangedPrVisibilityEvent, CreatedBuildEvent} from './build-events';
+import {BuildRetriever} from './build-retriever';
+import {BuildVerifier} from './build-verifier';
+import {respondWithError, throwRequestError} from './utils';
+
+const AIO_PREVIEW_JOB = 'aio_preview';
+
+// Interfaces - Types
+export interface PreviewServerConfig {
+  downloadsDir: string;
+  downloadSizeLimit: number;
+  buildArtifactPath: string;
+  buildsDir: string;
+  domainName: string;
+  githubOrg: string;
+  githubRepo: string;
+  githubTeamSlugs: string[];
+  circleCiToken: string;
+  githubToken: string;
+  significantFilesPattern: string;
+  trustedPrLabel: string;
+}
+
+const logger = createLogger('PreviewServer');
+
+// Classes
+export class PreviewServerFactory {
+  // Methods - Public
+  public static create(cfg: PreviewServerConfig): http.Server {
+    assertNotMissingOrEmpty('domainName', cfg.domainName);
+
+    const circleCiApi = new CircleCiApi(cfg.githubOrg, cfg.githubRepo, cfg.circleCiToken);
+    const githubApi = new GithubApi(cfg.githubToken);
+    const prs = new GithubPullRequests(githubApi, cfg.githubOrg, cfg.githubRepo);
+    const teams = new GithubTeams(githubApi, cfg.githubOrg);
+
+    const buildRetriever = new BuildRetriever(circleCiApi, cfg.downloadSizeLimit, cfg.downloadsDir);
+    const buildVerifier = new BuildVerifier(prs, teams, cfg.githubTeamSlugs, cfg.trustedPrLabel);
+    const buildCreator = PreviewServerFactory.createBuildCreator(prs, cfg.buildsDir, cfg.domainName);
+
+    const middleware = PreviewServerFactory.createMiddleware(buildRetriever, buildVerifier, buildCreator, cfg);
+    const httpServer = http.createServer(middleware as any);
+
+    httpServer.on('listening', () => {
+      const info = httpServer.address();
+      logger.info(`Up and running (and listening on ${info.address}:${info.port})...`);
+    });
+
+    return httpServer;
+  }
+
+  public static createMiddleware(buildRetriever: BuildRetriever, buildVerifier: BuildVerifier,
+                                 buildCreator: BuildCreator, cfg: PreviewServerConfig): express.Express {
+    const middleware = express();
+    const jsonParser = bodyParser.json();
+
+    // RESPOND TO IS-ALIVE PING
+    middleware.get(/^\/health-check\/?$/, (_req, res) => res.sendStatus(200));
+
+    // CIRCLE_CI BUILD COMPLETE WEBHOOK
+    middleware.post(/^\/circle-build\/?$/, jsonParser, async (req, res) => {
+      try {
+        if (!(
+          req.is('json') &&
+          req.body &&
+          req.body.payload &&
+          req.body.payload.build_num > 0 &&
+          req.body.payload.build_parameters &&
+          req.body.payload.build_parameters.CIRCLE_JOB
+        )) {
+          throwRequestError(400, `Incorrect body content. Expected JSON`, req);
+        }
+
+        const job = req.body.payload.build_parameters.CIRCLE_JOB;
+        const buildNum = req.body.payload.build_num;
+
+        logger.log(`Build:${buildNum}, Job:${job} - processing web-hook trigger`);
+
+        if (job !== AIO_PREVIEW_JOB) {
+          res.sendStatus(204);
+          logger.log(`Build:${buildNum}, Job:${job} -`,
+                     `Skipping preview processing because this is not the "${AIO_PREVIEW_JOB}" job.`);
+          return;
+        }
+
+        const { pr, sha, org, repo, success } = await buildRetriever.getGithubInfo(buildNum);
+
+        if (!success) {
+          res.sendStatus(204);
+          logger.log(`PR:${pr}, Build:${buildNum} - Skipping preview processing because this build did not succeed.`);
+          return;
+        }
+
+        assert(cfg.githubOrg === org,
+          `Invalid webhook: expected "githubOrg" property to equal "${cfg.githubOrg}" but got "${org}".`);
+        assert(cfg.githubRepo === repo,
+          `Invalid webhook: expected "githubRepo" property to equal "${cfg.githubRepo}" but got "${repo}".`);
+
+        // Do not deploy unless this PR has touched relevant files: `aio/` or `packages/` (except for spec files)
+        if (!await buildVerifier.getSignificantFilesChanged(pr, new RegExp(cfg.significantFilesPattern))) {
+          res.sendStatus(204);
+          logger.log(`PR:${pr}, Build:${buildNum} - ` +
+                     `Skipping preview processing because this PR did not touch any significant files.`);
+          return;
+        }
+
+        const artifactPath = await buildRetriever.downloadBuildArtifact(buildNum, pr, sha, cfg.buildArtifactPath);
+        const isPublic = await buildVerifier.getPrIsTrusted(pr);
+        await buildCreator.create(pr, sha, artifactPath, isPublic);
+        res.sendStatus(isPublic ? 201 : 202);
+      } catch (err) {
+        logger.error('CircleCI webhook error', err);
+        respondWithError(res, err);
+      }
+    });
+
+    // GITHUB PR UPDATED WEBHOOK
+    middleware.post(/^\/pr-updated\/?$/, jsonParser, async (req, res) => {
+      const { action, number: prNo }: { action?: string, number?: number } = req.body;
+      const visMayHaveChanged = !action || (action === 'labeled') || (action === 'unlabeled');
+
+      try {
+        if (!visMayHaveChanged) {
+          res.sendStatus(200);
+        } else if (!prNo) {
+          throwRequestError(400, `Missing or empty 'number' field`, req);
+        } else {
+          const isPublic = await buildVerifier.getPrIsTrusted(prNo);
+          await buildCreator.updatePrVisibility(prNo, isPublic);
+          res.sendStatus(200);
+        }
+      } catch (err) {
+        logger.error('PR update hook error', err);
+        respondWithError(res, err);
+      }
+    });
+
+    // ALL OTHER REQUESTS
+    middleware.all('*', req => throwRequestError(404, 'Unknown resource', req));
+    middleware.use((err: any, _req: any, res: express.Response, _next: any) => {
+      const statusText = http.STATUS_CODES[err.status] || '???';
+      logger.error(`Preview server error: ${err.status} - ${statusText}:`, err.message);
+      respondWithError(res, err);
+    });
+
+    return middleware;
+  }
+
+  public static createBuildCreator(prs: GithubPullRequests, buildsDir: string, domainName: string): BuildCreator {
+    const buildCreator = new BuildCreator(buildsDir);
+    const postPreviewsComment = (pr: number, shas: string[]) => {
+      const body = shas.
+        map(sha => `You can preview ${sha} at https://pr${pr}-${sha}.${domainName}/.`).
+        join('\n');
+
+      return prs.addComment(pr, body);
+    };
+
+    buildCreator.on(CreatedBuildEvent.type, ({pr, sha, isPublic}: CreatedBuildEvent) => {
+      if (isPublic) {
+        postPreviewsComment(pr, [sha]);
+      }
+    });
+
+    buildCreator.on(ChangedPrVisibilityEvent.type, ({pr, shas, isPublic}: ChangedPrVisibilityEvent) => {
+      if (isPublic && shas.length) {
+        postPreviewsComment(pr, shas);
+      }
+    });
+
+    return buildCreator;
+  }
+}

aio/aio-builds-setup/dockerbuild/scripts-js/lib/preview-server/utils.ts

@@ -0,0 +1,29 @@
+import * as express from 'express';
+import {promisify} from 'util';
+import {PreviewServerError} from './preview-error';
+
+/**
+ * Update the response to report that an error has occurred.
+ * @param res The response to configure as an error.
+ * @param err The error that needs to be reported.
+ */
+export async function respondWithError(res: express.Response, err: any): Promise<void> {
+  if (!(err instanceof PreviewServerError)) {
+    err = new PreviewServerError(500, String((err && err.message) || err));
+  }
+
+  res.status(err.status);
+  await promisify(res.end.bind(res))(err.message);
+}
+
+/**
+ * Throw an exception that describes the given error information.
+ * @param status The HTTP status code include in the error.
+ * @param error The error message to include in the error.
+ * @param req The request that triggered this error.
+ */
+export function throwRequestError(status: number, error: string, req: express.Request): never {
+  const message = `${error} in request: ${req.method} ${req.originalUrl}` +
+                  (!req.body ? '' : ` ${JSON.stringify(req.body)}`);
+  throw new PreviewServerError(status, message);
+}