feat(aio): enable previews for any PR

This commit introduces the ability to show previews for PRs by any author. It works as follows: - The build artifacts of all PRs are uploaded to the preview server. - Automatically verified PRs (i.e. from trusted authors or having a specific label) are deployed and publicly accessible as usual. - PRs that could not be automatically verified are stored for later use (after re-verification). - A PR can be marked as "trusted" and make its preview publicly accessible by adding the GitHub label specified in the `AIO_TRUSTED_PR_LABEL` env var of the preview server. At the moment, there is no automatic mechanism for notifying the preview server about changes to the PR's verification status. The PR's "visibility" will be checked and updated every time a new build is uploaded.
2017-06-19 01:15:07 +03:00
parent 0fe685102f
commit 8ae0eec230
20 changed files with 915 additions and 300 deletions
--- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-creator.ts
+++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-creator.ts
@ -5,11 +5,14 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as shell from 'shelljs';
 import {assertNotMissingOrEmpty} from '../common/utils';
-import {CreatedBuildEvent} from './build-events';
+import {ChangedPrVisibilityEvent, CreatedBuildEvent} from './build-events';
 import {UploadError} from './upload-error';

 // Classes
 export class BuildCreator extends EventEmitter {
+  // Properties - Public, Static
+  public static HIDDEN_DIR_PREFIX = 'hidden--';
+
  // Constructor
  constructor(protected buildsDir: string) {
    super();
@ -17,13 +20,43 @@ export class BuildCreator extends EventEmitter {
  }

  // Methods - Public
-  public create(pr: string, sha: string, archivePath: string): Promise<any> {
-    const prDir = path.join(this.buildsDir, pr);
+  public changePrVisibility(pr: string, makePublic: boolean): Promise<void> {
+    const {oldPrDir, newPrDir} = this.getCandidatePrDirs(pr, makePublic);
+
+    return Promise.
+      all([this.exists(oldPrDir), this.exists(newPrDir)]).
+      then(([oldPrDirExisted, newPrDirExisted]) => {
+        if (!oldPrDirExisted) {
+          throw new UploadError(404, `Request to move non-existing directory '${oldPrDir}' to '${newPrDir}'.`);
+        } else if (newPrDirExisted) {
+          throw new UploadError(409, `Request to move '${oldPrDir}' to existing directory '${newPrDir}'.`);
+        }
+
+        return Promise.resolve().
+          then(() => shell.mv(oldPrDir, newPrDir)).
+          then(() => this.listShasByDate(newPrDir)).
+          then(shas => this.emit(ChangedPrVisibilityEvent.type, new ChangedPrVisibilityEvent(+pr, shas, makePublic))).
+          then(() => undefined);
+      }).
+      catch(err => {
+        if (!(err instanceof UploadError)) {
+          err = new UploadError(500, `Error while making PR ${pr} ${makePublic ? 'public' : 'hidden'}.\n${err}`);
+        }
+
+        throw err;
+      });
+  }
+
+  public create(pr: string, sha: string, archivePath: string, isPublic: boolean): Promise<void> {
+    const {oldPrDir: otherVisPrDir, newPrDir: prDir} = this.getCandidatePrDirs(pr, isPublic);
    const shaDir = path.join(prDir, sha);
    let dirToRemoveOnError: string;

-    return Promise.
-      all([this.exists(prDir), this.exists(shaDir)]).
+    return Promise.resolve().
+      then(() => this.exists(otherVisPrDir)).
+      // If the same PR exists with different visibility, update the visibility first.
+      then(otherVisPrDirExisted => (otherVisPrDirExisted && this.changePrVisibility(pr, isPublic)) as any).
+      then(() => Promise.all([this.exists(prDir), this.exists(shaDir)])).
      then(([prDirExisted, shaDirExisted]) => {
        if (shaDirExisted) {
          throw new UploadError(409, `Request to overwrite existing directory: ${shaDir}`);
@ -34,7 +67,8 @@ export class BuildCreator extends EventEmitter {
        return Promise.resolve().
          then(() => shell.mkdir('-p', shaDir)).
          then(() => this.extractArchive(archivePath, shaDir)).
-          then(() => this.emit(CreatedBuildEvent.type, new CreatedBuildEvent(+pr, sha)));
+          then(() => this.emit(CreatedBuildEvent.type, new CreatedBuildEvent(+pr, sha, isPublic))).
+          then(() => undefined);
      }).
      catch(err => {
        if (dirToRemoveOnError) {
@ -78,4 +112,26 @@ export class BuildCreator extends EventEmitter {
      });
    });
  }
+
+  protected getCandidatePrDirs(pr: string, isPublic: boolean) {
+    const hiddenPrDir = path.join(this.buildsDir, BuildCreator.HIDDEN_DIR_PREFIX + pr);
+    const publicPrDir = path.join(this.buildsDir, pr);
+
+    const oldPrDir = isPublic ? hiddenPrDir : publicPrDir;
+    const newPrDir = isPublic ? publicPrDir : hiddenPrDir;
+
+    return {oldPrDir, newPrDir};
+  }
+
+  protected listShasByDate(inputDir: string): Promise<string[]> {
+    return Promise.resolve().
+      then(() => shell.ls('-l', inputDir) as any as Promise<(fs.Stats & {name: string})[]>).
+      // Keep directories only.
+      // (Also, convert to standard Array - ShellJS provides custom `sort()` method for sorting file contents.)
+      then(items => items.filter(item => item.isDirectory())).
+      // Sort by modification date.
+      then(items => items.sort((a, b) => a.mtime.getTime() - b.mtime.getTime())).
+      // Return directory names.
+      then(items => items.map(item => item.name));
+  }
 }
--- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-events.ts
+++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-events.ts
@ -1,8 +1,16 @@
 // Classes
+export class ChangedPrVisibilityEvent {
+  // Properties - Public, Static
+  public static type = 'pr.changedVisibility';
+
+  // Constructor
+  constructor(public pr: number, public shas: string[], public isPublic: boolean) {}
+}
+
 export class CreatedBuildEvent {
  // Properties - Public, Static
  public static type = 'build.created';

  // Constructor
-  constructor(public pr: number, public sha: string) {}
+  constructor(public pr: number, public sha: string, public isPublic: boolean) {}
 }
--- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-verifier.ts
+++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-verifier.ts
@ -1,6 +1,6 @@
 // Imports
 import * as jwt from 'jsonwebtoken';
-import {GithubPullRequests} from '../common/github-pull-requests';
+import {GithubPullRequests, PullRequest} from '../common/github-pull-requests';
 import {GithubTeams} from '../common/github-teams';
 import {assertNotMissingOrEmpty} from '../common/utils';
 import {UploadError} from './upload-error';
@ -11,6 +11,12 @@ interface JwtPayload {
  'pull-request': number;
 }

+// Enums
+export enum BUILD_VERIFICATION_STATUS {
+  verifiedAndTrusted,
+  verifiedNotTrusted,
+}
+
 // Classes
 export class BuildVerifier {
  // Properties - Protected
@ -19,27 +25,27 @@ export class BuildVerifier {

  // Constructor
  constructor(protected secret: string, githubToken: string, protected repoSlug: string, organization: string,
-              protected allowedTeamSlugs: string[]) {
+              protected allowedTeamSlugs: string[], protected trustedPrLabel: string) {
    assertNotMissingOrEmpty('secret', secret);
    assertNotMissingOrEmpty('githubToken', githubToken);
    assertNotMissingOrEmpty('repoSlug', repoSlug);
    assertNotMissingOrEmpty('organization', organization);
    assertNotMissingOrEmpty('allowedTeamSlugs', allowedTeamSlugs && allowedTeamSlugs.join(''));
+    assertNotMissingOrEmpty('trustedPrLabel', trustedPrLabel);

    this.githubPullRequests = new GithubPullRequests(githubToken, repoSlug);
    this.githubTeams = new GithubTeams(githubToken, organization);
  }

  // Methods - Public
-  public getPrAuthorTeamMembership(pr: number): Promise<{author: string, isMember: boolean}> {
+  public getPrIsTrusted(pr: number): Promise<boolean> {
    return Promise.resolve().
      then(() => this.githubPullRequests.fetch(pr)).
-      then(prInfo => prInfo.user.login).
-      then(author => this.githubTeams.isMemberBySlug(author, this.allowedTeamSlugs).
-        then(isMember => ({author, isMember})));
+      then(prInfo => this.hasLabel(prInfo, this.trustedPrLabel) ||
+                     this.githubTeams.isMemberBySlug(prInfo.user.login, this.allowedTeamSlugs));
  }

-  public verify(expectedPr: number, authHeader: string): Promise<void> {
+  public verify(expectedPr: number, authHeader: string): Promise<BUILD_VERIFICATION_STATUS> {
    return Promise.resolve().
      then(() => this.extractJwtString(authHeader)).
      then(jwtString => this.verifyJwt(expectedPr, jwtString)).
@ -52,6 +58,10 @@ export class BuildVerifier {
    return input.replace(/^token +/i, '');
  }

+  protected hasLabel(prInfo: PullRequest, label: string) {
+    return prInfo.labels.some(labelObj => labelObj.name === label);
+  }
+
  protected verifyJwt(expectedPr: number, token: string): Promise<JwtPayload> {
    return new Promise((resolve, reject) => {
      jwt.verify(token, this.secret, {issuer: 'Travis CI, GmbH'}, (err, payload: JwtPayload) => {
@ -68,11 +78,10 @@ export class BuildVerifier {
    });
  }

-  protected verifyPr(pr: number): Promise<void> {
-    return this.getPrAuthorTeamMembership(pr).
-      then(({author, isMember}) => isMember ? Promise.resolve() : Promise.reject(
-        `User '${author}' is not an active member of any of the following teams: ` +
-        `${this.allowedTeamSlugs.join(', ')}`,
-      ));
+  protected verifyPr(pr: number): Promise<BUILD_VERIFICATION_STATUS> {
+    return this.getPrIsTrusted(pr).
+      then(isTrusted => Promise.resolve(isTrusted ?
+          BUILD_VERIFICATION_STATUS.verifiedAndTrusted :
+          BUILD_VERIFICATION_STATUS.verifiedNotTrusted));
  }
 }
--- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/index-preverify-pr.ts
+++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/index-preverify-pr.ts
@ -12,28 +12,28 @@ function _main() {
  const repoSlug = getEnvVar('AIO_REPO_SLUG');
  const organization = getEnvVar('AIO_GITHUB_ORGANIZATION');
  const allowedTeamSlugs = getEnvVar('AIO_GITHUB_TEAM_SLUGS').split(',');
+  const trustedPrLabel = getEnvVar('AIO_TRUSTED_PR_LABEL');
  const pr = +getEnvVar('AIO_PREVERIFY_PR');

-  const buildVerifier = new BuildVerifier(secret, githubToken, repoSlug, organization, allowedTeamSlugs);
+  const buildVerifier = new BuildVerifier(secret, githubToken, repoSlug, organization, allowedTeamSlugs,
+                                          trustedPrLabel);

  // Exit codes:
-  // - 0: The PR author is a member.
+  // - 0: The PR can be automatically trusted (i.e. author belongs to trusted team or PR has the "trusted PR" label).
  // - 1: An error occurred.
-  // - 2: The PR author is not a member.
-  buildVerifier.getPrAuthorTeamMembership(pr).
-    then(({author, isMember}) => {
-      if (isMember) {
-        process.exit(0);
-      } else {
-        const errorMessage = `User '${author}' is not an active member of any of the following teams: ` +
-                             `${allowedTeamSlugs.join(', ')}`;
-        onError(errorMessage, 2);
+  // - 2: The PR cannot be automatically trusted.
+  buildVerifier.getPrIsTrusted(pr).
+    then(isTrusted => {
+      if (!isTrusted) {
+        console.warn(
+            `The PR cannot be automatically verified, because it doesn't have the "${trustedPrLabel}" label and the ` +
+            `the author is not an active member of any of the following teams: ${allowedTeamSlugs.join(', ')}`);
      }
-    }).
-    catch(err => onError(err, 1));
-}

-function onError(err: string, exitCode: number) {
-  console.error(err);
-  process.exit(exitCode || 1);
+      process.exit(isTrusted ? 0 : 2);
+    }).
+    catch(err => {
+      console.error(err);
+      process.exit(1);
+    });
 }
--- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/index-test.ts
+++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/index-test.ts
@ -1,10 +1,10 @@
 // Imports
 import {GithubPullRequests} from '../common/github-pull-requests';
-import {BuildVerifier} from './build-verifier';
+import {BUILD_VERIFICATION_STATUS, BuildVerifier} from './build-verifier';

 // Run
 // TODO(gkalpak): Add e2e tests to cover these interactions as well.
 GithubPullRequests.prototype.addComment = () => Promise.resolve();
-BuildVerifier.prototype.verify = () => Promise.resolve();
+BuildVerifier.prototype.verify = () => Promise.resolve(BUILD_VERIFICATION_STATUS.verifiedAndTrusted);
 // tslint:disable-next-line: no-var-requires
 require('./index');
--- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/index.ts
+++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/index.ts
@ -10,6 +10,7 @@ const AIO_GITHUB_TEAM_SLUGS = getEnvVar('AIO_GITHUB_TEAM_SLUGS');
 const AIO_GITHUB_TOKEN = getEnvVar('AIO_GITHUB_TOKEN');
 const AIO_PREVIEW_DEPLOYMENT_TOKEN = getEnvVar('AIO_PREVIEW_DEPLOYMENT_TOKEN');
 const AIO_REPO_SLUG = getEnvVar('AIO_REPO_SLUG');
+const AIO_TRUSTED_PR_LABEL = getEnvVar('AIO_TRUSTED_PR_LABEL');
 const AIO_UPLOAD_HOSTNAME = getEnvVar('AIO_UPLOAD_HOSTNAME');
 const AIO_UPLOAD_PORT = +getEnvVar('AIO_UPLOAD_PORT');
 const AIO_WWW_USER = getEnvVar('AIO_WWW_USER');
@ -29,6 +30,7 @@ function _main() {
      githubToken: AIO_GITHUB_TOKEN,
      repoSlug: AIO_REPO_SLUG,
      secret: AIO_PREVIEW_DEPLOYMENT_TOKEN,
+      trustedPrLabel: AIO_TRUSTED_PR_LABEL,
    }).
    listen(AIO_UPLOAD_PORT, AIO_UPLOAD_HOSTNAME);
 }
--- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/upload-server-factory.ts
+++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/upload-server-factory.ts
@ -4,8 +4,8 @@ import * as http from 'http';
 import {GithubPullRequests} from '../common/github-pull-requests';
 import {assertNotMissingOrEmpty} from '../common/utils';
 import {BuildCreator} from './build-creator';
-import {CreatedBuildEvent} from './build-events';
-import {BuildVerifier} from './build-verifier';
+import {ChangedPrVisibilityEvent, CreatedBuildEvent} from './build-events';
+import {BUILD_VERIFICATION_STATUS, BuildVerifier} from './build-verifier';
 import {UploadError} from './upload-error';

 // Constants
@ -21,6 +21,7 @@ interface UploadServerConfig {
  githubToken: string;
  repoSlug: string;
  secret: string;
+  trustedPrLabel: string;
 }

 // Classes
@ -34,10 +35,12 @@ class UploadServerFactory {
    githubToken,
    repoSlug,
    secret,
+    trustedPrLabel,
  }: UploadServerConfig): http.Server {
    assertNotMissingOrEmpty('domainName', domainName);

-    const buildVerifier = new BuildVerifier(secret, githubToken, repoSlug, githubOrganization, githubTeamSlugs);
+    const buildVerifier = new BuildVerifier(secret, githubToken, repoSlug, githubOrganization, githubTeamSlugs,
+                                            trustedPrLabel);
    const buildCreator = this.createBuildCreator(buildsDir, githubToken, repoSlug, domainName);

    const middleware = this.createMiddleware(buildVerifier, buildCreator);
@ -56,12 +59,24 @@ class UploadServerFactory {
                               domainName: string): BuildCreator {
    const buildCreator = new BuildCreator(buildsDir);
    const githubPullRequests = new GithubPullRequests(githubToken, repoSlug);
+    const postPreviewsComment = (pr: number, shas: string[]) => {
+      const body = shas.
+        map(sha => `You can preview ${sha} at https://pr${pr}-${sha}.${domainName}/.`).
+        join('\n');

-    buildCreator.on(CreatedBuildEvent.type, ({pr, sha}: CreatedBuildEvent) => {
-      const body = `The angular.io preview for ${sha} is available [here][1].\n\n` +
-                   `[1]: https://pr${pr}-${sha}.${domainName}/`;
+      return githubPullRequests.addComment(pr, body);
+    };

-      githubPullRequests.addComment(pr, body);
+    buildCreator.on(CreatedBuildEvent.type, ({pr, sha, isPublic}: CreatedBuildEvent) => {
+      if (isPublic) {
+        postPreviewsComment(pr, [sha]);
+      }
+    });
+
+    buildCreator.on(ChangedPrVisibilityEvent.type, ({pr, shas, isPublic}: ChangedPrVisibilityEvent) => {
+      if (isPublic && shas.length) {
+        postPreviewsComment(pr, shas);
+      }
    });

    return buildCreator;
@ -83,8 +98,9 @@ class UploadServerFactory {
      } else {
        buildVerifier.
          verify(+pr, authHeader).
-          then(() => buildCreator.create(pr, sha, archive)).
-          then(() => res.sendStatus(201)).
+          then(verStatus => verStatus === BUILD_VERIFICATION_STATUS.verifiedAndTrusted).
+          then(isPublic => buildCreator.create(pr, sha, archive, isPublic).
+            then(() => res.sendStatus(isPublic ? 201 : 202))).
          catch(err => this.respondWithError(res, err));
      }
    });