diff --git a/.circleci/config.yml b/.circleci/config.yml index 6350a32324..512acc971a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -292,4 +292,4 @@ workflows: - master notify: webhooks: - - url: https://35.224.48.224/circle-build \ No newline at end of file + - url: https://ngbuilds.io/circle-build \ No newline at end of file diff --git a/aio/aio-builds-setup/dockerbuild/Dockerfile b/aio/aio-builds-setup/dockerbuild/Dockerfile index e12bc29536..41887f4a0b 100644 --- a/aio/aio-builds-setup/dockerbuild/Dockerfile +++ b/aio/aio-builds-setup/dockerbuild/Dockerfile @@ -8,6 +8,7 @@ LABEL name="angular.io PR preview" \ VOLUME /aio-secrets VOLUME /var/www/aio-builds +VOLUME /dockerbuild EXPOSE 80 443 @@ -22,7 +23,9 @@ ARG TEST_AIO_BUILDS_DIR=/tmp/aio-builds ARG AIO_DOMAIN_NAME=ngbuilds.io ARG TEST_AIO_DOMAIN_NAME=$AIO_DOMAIN_NAME.localhost ARG AIO_GITHUB_ORGANIZATION=angular -ARG TEST_AIO_GITHUB_ORGANIZATION=angular +ARG TEST_AIO_GITHUB_ORGANIZATION=test-org +ARG AIO_GITHUB_REPO=angular +ARG TEST_AIO_GITHUB_REPO=test-repo ARG AIO_GITHUB_TEAM_SLUGS=team,aio-contributors ARG TEST_AIO_GITHUB_TEAM_SLUGS=team,aio-contributors ARG AIO_NGINX_HOSTNAME=$AIO_DOMAIN_NAME @@ -31,34 +34,36 @@ ARG AIO_NGINX_PORT_HTTP=80 ARG TEST_AIO_NGINX_PORT_HTTP=8080 ARG AIO_NGINX_PORT_HTTPS=443 ARG TEST_AIO_NGINX_PORT_HTTPS=4433 -ARG AIO_REPO_SLUG=angular/angular -ARG TEST_AIO_REPO_SLUG=test-repo/test-slug +ARG AIO_SIGNIFICANT_FILES_PATTERN='^(?:aio|packages)/(?!.*[._]spec\\.[jt]s$)' +ARG TEST_AIO_SIGNIFICANT_FILES_PATTERN=$AIO_SIGNIFICANT_FILES_PATTERN ARG AIO_TRUSTED_PR_LABEL="aio: preview" ARG TEST_AIO_TRUSTED_PR_LABEL="aio: preview" ARG AIO_UPLOAD_HOSTNAME=upload.localhost ARG TEST_AIO_UPLOAD_HOSTNAME=upload.localhost ARG AIO_UPLOAD_MAX_SIZE=20971520 -ARG TEST_AIO_UPLOAD_MAX_SIZE=20971520 +ARG TEST_AIO_UPLOAD_MAX_SIZE=200 ARG AIO_UPLOAD_PORT=3000 ARG TEST_AIO_UPLOAD_PORT=3001 -ENV AIO_BUILDS_DIR=$AIO_BUILDS_DIR TEST_AIO_BUILDS_DIR=$TEST_AIO_BUILDS_DIR \ - AIO_DOMAIN_NAME=$AIO_DOMAIN_NAME TEST_AIO_DOMAIN_NAME=$TEST_AIO_DOMAIN_NAME \ - AIO_GITHUB_ORGANIZATION=$AIO_GITHUB_ORGANIZATION TEST_AIO_GITHUB_ORGANIZATION=$TEST_AIO_GITHUB_ORGANIZATION \ - AIO_GITHUB_TEAM_SLUGS=$AIO_GITHUB_TEAM_SLUGS TEST_AIO_GITHUB_TEAM_SLUGS=$TEST_AIO_GITHUB_TEAM_SLUGS \ - AIO_LOCALCERTS_DIR=/etc/ssl/localcerts TEST_AIO_LOCALCERTS_DIR=/etc/ssl/localcerts-test \ - AIO_NGINX_HOSTNAME=$AIO_NGINX_HOSTNAME TEST_AIO_NGINX_HOSTNAME=$TEST_AIO_NGINX_HOSTNAME \ - AIO_NGINX_LOGS_DIR=/var/log/aio/nginx TEST_AIO_NGINX_LOGS_DIR=/var/log/aio/nginx-test \ - AIO_NGINX_PORT_HTTP=$AIO_NGINX_PORT_HTTP TEST_AIO_NGINX_PORT_HTTP=$TEST_AIO_NGINX_PORT_HTTP \ - AIO_NGINX_PORT_HTTPS=$AIO_NGINX_PORT_HTTPS TEST_AIO_NGINX_PORT_HTTPS=$TEST_AIO_NGINX_PORT_HTTPS \ - AIO_REPO_SLUG=$AIO_REPO_SLUG TEST_AIO_REPO_SLUG=$TEST_AIO_REPO_SLUG \ - AIO_SCRIPTS_JS_DIR=/usr/share/aio-scripts-js \ - AIO_SCRIPTS_SH_DIR=/usr/share/aio-scripts-sh \ - AIO_TRUSTED_PR_LABEL=$AIO_TRUSTED_PR_LABEL TEST_AIO_TRUSTED_PR_LABEL=$TEST_AIO_TRUSTED_PR_LABEL \ - AIO_UPLOAD_HOSTNAME=$AIO_UPLOAD_HOSTNAME TEST_AIO_UPLOAD_HOSTNAME=$TEST_AIO_UPLOAD_HOSTNAME \ - AIO_UPLOAD_MAX_SIZE=$AIO_UPLOAD_MAX_SIZE TEST_AIO_UPLOAD_MAX_SIZE=$TEST_AIO_UPLOAD_MAX_SIZE \ - AIO_UPLOAD_PORT=$AIO_UPLOAD_PORT TEST_AIO_UPLOAD_PORT=$TEST_AIO_UPLOAD_PORT \ - AIO_WWW_USER=www-data \ +ENV AIO_ARTIFACT_PATH=$AIO_ARTIFACT_PATH TEST_AIO_ARTIFACT_PATH=$TEST_AIO_ARTIFACT_PATH \ + AIO_BUILDS_DIR=$AIO_BUILDS_DIR TEST_AIO_BUILDS_DIR=$TEST_AIO_BUILDS_DIR \ + AIO_DOMAIN_NAME=$AIO_DOMAIN_NAME TEST_AIO_DOMAIN_NAME=$TEST_AIO_DOMAIN_NAME \ + AIO_GITHUB_ORGANIZATION=$AIO_GITHUB_ORGANIZATION TEST_AIO_GITHUB_ORGANIZATION=$TEST_AIO_GITHUB_ORGANIZATION \ + AIO_GITHUB_REPO=$AIO_GITHUB_REPO TEST_AIO_GITHUB_REPO=$TEST_AIO_GITHUB_REPO \ + AIO_GITHUB_TEAM_SLUGS=$AIO_GITHUB_TEAM_SLUGS TEST_AIO_GITHUB_TEAM_SLUGS=$TEST_AIO_GITHUB_TEAM_SLUGS \ + AIO_LOCALCERTS_DIR=/etc/ssl/localcerts TEST_AIO_LOCALCERTS_DIR=/etc/ssl/localcerts-test \ + AIO_NGINX_HOSTNAME=$AIO_NGINX_HOSTNAME TEST_AIO_NGINX_HOSTNAME=$TEST_AIO_NGINX_HOSTNAME \ + AIO_NGINX_LOGS_DIR=/var/log/aio/nginx TEST_AIO_NGINX_LOGS_DIR=/var/log/aio/nginx-test \ + AIO_NGINX_PORT_HTTP=$AIO_NGINX_PORT_HTTP TEST_AIO_NGINX_PORT_HTTP=$TEST_AIO_NGINX_PORT_HTTP \ + AIO_NGINX_PORT_HTTPS=$AIO_NGINX_PORT_HTTPS TEST_AIO_NGINX_PORT_HTTPS=$TEST_AIO_NGINX_PORT_HTTPS \ + AIO_SCRIPTS_JS_DIR=/usr/share/aio-scripts-js \ + AIO_SCRIPTS_SH_DIR=/usr/share/aio-scripts-sh \ + AIO_SIGNIFICANT_FILES_PATTERN=$AIO_SIGNIFICANT_FILES_PATTERN TEST_AIO_SIGNIFICANT_FILES_PATTERN=$TEST_AIO_SIGNIFICANT_FILES_PATTERN \ + AIO_TRUSTED_PR_LABEL=$AIO_TRUSTED_PR_LABEL TEST_AIO_TRUSTED_PR_LABEL=$TEST_AIO_TRUSTED_PR_LABEL \ + AIO_UPLOAD_HOSTNAME=$AIO_UPLOAD_HOSTNAME TEST_AIO_UPLOAD_HOSTNAME=$TEST_AIO_UPLOAD_HOSTNAME \ + AIO_UPLOAD_MAX_SIZE=$AIO_UPLOAD_MAX_SIZE TEST_AIO_UPLOAD_MAX_SIZE=$TEST_AIO_UPLOAD_MAX_SIZE \ + AIO_UPLOAD_PORT=$AIO_UPLOAD_PORT TEST_AIO_UPLOAD_PORT=$TEST_AIO_UPLOAD_PORT \ + AIO_WWW_USER=www-data \ NODE_ENV=production diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/clean-up/build-cleaner.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/clean-up/build-cleaner.ts index a5fef4bb4b..5108b44cbd 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/clean-up/build-cleaner.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/clean-up/build-cleaner.ts @@ -3,29 +3,51 @@ import * as fs from 'fs'; import * as path from 'path'; import * as shell from 'shelljs'; import {HIDDEN_DIR_PREFIX} from '../common/constants'; +import {GithubApi} from '../common/github-api'; import {GithubPullRequests} from '../common/github-pull-requests'; -import {assertNotMissingOrEmpty} from '../common/utils'; +import {assertNotMissingOrEmpty, getPrInfoFromDownloadPath} from '../common/utils'; // Classes export class BuildCleaner { + // Constructor - constructor(protected buildsDir: string, protected repoSlug: string, protected githubToken: string) { + constructor(protected buildsDir: string, protected githubOrg: string, protected githubRepo: string, + protected githubToken: string, protected downloadsDir: string, protected artifactPath: string) { assertNotMissingOrEmpty('buildsDir', buildsDir); - assertNotMissingOrEmpty('repoSlug', repoSlug); + assertNotMissingOrEmpty('githubOrg', githubOrg); + assertNotMissingOrEmpty('githubRepo', githubRepo); assertNotMissingOrEmpty('githubToken', githubToken); + assertNotMissingOrEmpty('downloadsDir', downloadsDir); + assertNotMissingOrEmpty('artifactPath', artifactPath); } // Methods - Public - public cleanUp(): Promise { - return Promise.all([ - this.getExistingBuildNumbers(), - this.getOpenPrNumbers(), - ]).then(([existingBuilds, openPrs]) => this.removeUnnecessaryBuilds(existingBuilds, openPrs)); + public async cleanUp() { + try { + this.logger.log('Cleaning up builds and downloads'); + const openPrs = await this.getOpenPrNumbers(); + this.logger.log(`Open pull requests: ${openPrs.length}`); + await Promise.all([ + this.cleanBuilds(openPrs), + this.cleanDownloads(openPrs), + ]); + } catch (error) { + this.logger.error('ERROR:', error); + } } - // Methods - Protected - protected getExistingBuildNumbers(): Promise { - return new Promise((resolve, reject) => { + public async cleanBuilds(openPrs: number[]) { + const existingBuilds = await this.getExistingBuildNumbers(); + await this.removeUnnecessaryBuilds(existingBuilds, openPrs); + } + + public async cleanDownloads(openPrs: number[]) { + const existingDownloads = await this.getExistingDownloads(); + await this.removeUnnecessaryDownloads(existingDownloads, openPrs); + } + + public getExistingBuildNumbers() { + return new Promise((resolve, reject) => { fs.readdir(this.buildsDir, (err, files) => { if (err) { return reject(err); @@ -41,15 +63,14 @@ export class BuildCleaner { }); } - protected getOpenPrNumbers(): Promise { - const githubPullRequests = new GithubPullRequests(this.githubToken, this.repoSlug); - - return githubPullRequests. - fetchAll('open'). - then(prs => prs.map(pr => pr.number)); + public async getOpenPrNumbers() { + const api = new GithubApi(this.githubToken); + const githubPullRequests = new GithubPullRequests(api, this.githubOrg, this.githubRepo); + const prs = await githubPullRequests.fetchAll('open'); + return prs.map(pr => pr.number); } - protected removeDir(dir: string) { + public removeDir(dir: string) { try { if (shell.test('-d', dir)) { shell.chmod('-R', 'a+w', dir); @@ -60,11 +81,10 @@ export class BuildCleaner { } } - protected removeUnnecessaryBuilds(existingBuildNumbers: number[], openPrNumbers: number[]) { + public removeUnnecessaryBuilds(existingBuildNumbers: number[], openPrNumbers: number[]) { const toRemove = existingBuildNumbers.filter(num => !openPrNumbers.includes(num)); console.log(`Existing builds: ${existingBuildNumbers.length}`); - console.log(`Open pull requests: ${openPrNumbers.length}`); console.log(`Removing ${toRemove.length} build(s): ${toRemove.join(', ')}`); // Try removing public dirs. @@ -77,4 +97,29 @@ export class BuildCleaner { map(num => path.join(this.buildsDir, HIDDEN_DIR_PREFIX + String(num))). forEach(dir => this.removeDir(dir)); } + + public getExistingDownloads() { + const artifactFile = path.basename(this.artifactPath); + return new Promise((resolve, reject) => { + fs.readdir(this.downloadsDir, (err, files) => { + if (err) { + return reject(err); + } + files = files.filter(file => file.endsWith(artifactFile)); + resolve(files); + }); + }); + } + + public removeUnnecessaryDownloads(existingDownloads: string[], openPrNumbers: number[]) { + const toRemove = existingDownloads.filter(filePath => { + const {pr} = getPrInfoFromDownloadPath(filePath); + return !openPrNumbers.includes(pr); + }); + + console.log(`Existing downloads: ${existingDownloads.length}`); + console.log(`Removing ${toRemove.length} download(s): ${toRemove.join(', ')}`); + + toRemove.forEach(filePath => shell.rm(filePath)); + } } diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/clean-up/index.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/clean-up/index.ts index c9819dd998..0996861ba7 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/clean-up/index.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/clean-up/index.ts @@ -1,12 +1,14 @@ // Imports -import {getEnvVar} from '../common/utils'; +import {AIO_DOWNLOADS_DIR} from '../common/constants'; +import { + AIO_ARTIFACT_PATH, + AIO_BUILDS_DIR, + AIO_GITHUB_ORGANIZATION, + AIO_GITHUB_REPO, + AIO_GITHUB_TOKEN, +} from '../common/env-variables'; import {BuildCleaner} from './build-cleaner'; -// Constants -const AIO_BUILDS_DIR = getEnvVar('AIO_BUILDS_DIR'); -const AIO_GITHUB_TOKEN = getEnvVar('AIO_GITHUB_TOKEN', true); -const AIO_REPO_SLUG = getEnvVar('AIO_REPO_SLUG'); - // Run _main(); @@ -14,7 +16,13 @@ _main(); function _main() { console.log(`[${new Date()}] - Cleaning up builds...`); - const buildCleaner = new BuildCleaner(AIO_BUILDS_DIR, AIO_REPO_SLUG, AIO_GITHUB_TOKEN); + const buildCleaner = new BuildCleaner( + AIO_BUILDS_DIR, + AIO_GITHUB_ORGANIZATION, + AIO_GITHUB_REPO, + AIO_GITHUB_TOKEN, + AIO_DOWNLOADS_DIR, + AIO_ARTIFACT_PATH); buildCleaner.cleanUp().catch(err => { console.error('ERROR:', err); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/circle-ci-api.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/circle-ci-api.ts new file mode 100644 index 0000000000..22fd9ced83 --- /dev/null +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/circle-ci-api.ts @@ -0,0 +1,90 @@ +// Imports +import fetch from 'node-fetch'; +import {assertNotMissingOrEmpty} from './utils'; + +// Constants +const CIRCLE_CI_API_URL = 'https://circleci.com/api/v1.1/project/github'; + +// Interfaces - Types +export interface ArtifactInfo { + path: string; + pretty_path: string; + node_index: number; + url: string; +} + +export type ArtifactResponse = ArtifactInfo[]; + +export interface BuildInfo { + reponame: string; + failed: boolean; + branch: string; + username: string; + build_num: number; + has_artifacts: boolean; + outcome: string; // e.g. 'success' + vcs_revision: string; // HEAD SHA + // there are other fields but they are not used in this code +} + +/** + * A Helper that can interact with the CircleCI API. + */ +export class CircleCiApi { + + private tokenParam = `circle-token=${this.circleCiToken}`; + + /** + * Construct a helper that can interact with the CircleCI REST API. + * @param githubOrg The Github organisation whose repos we want to access in CircleCI (e.g. angular). + * @param githubRepo The Github repo whose builds we want to access in CircleCI (e.g. angular). + * @param circleCiToken The CircleCI API access token (secret). + */ + constructor( + private githubOrg: string, + private githubRepo: string, + private circleCiToken: string, + ) { + assertNotMissingOrEmpty('githubOrg', githubOrg); + assertNotMissingOrEmpty('githubRepo', githubRepo); + assertNotMissingOrEmpty('circleCiToken', circleCiToken); + } + + /** + * Get the info for a build from the CircleCI API + * @param buildNumber The CircleCI build number that generated the artifact. + * @returns A promise to the info about the build + */ + public async getBuildInfo(buildNumber: number) { + try { + const baseUrl = `${CIRCLE_CI_API_URL}/${this.githubOrg}/${this.githubRepo}/${buildNumber}`; + const response = await fetch(`${baseUrl}?${this.tokenParam}`); + if (response.status !== 200) { + throw new Error(`${baseUrl}: ${response.status} - ${response.statusText}`); + } + return response.json(); + } catch (error) { + throw new Error(`CircleCI build info request failed (${error.message})`); + } + } + + /** + * Query the CircleCI API to get a URL for a specified artifact from a specified build. + * @param artifactPath The path, within the build to the artifact. + * @returns A promise to the URL that can be requested to download the actual build artifact file. + */ + public async getBuildArtifactUrl(buildNumber: number, artifactPath: string) { + const baseUrl = `${CIRCLE_CI_API_URL}/${this.githubOrg}/${this.githubRepo}/${buildNumber}`; + try { + const response = await fetch(`${baseUrl}/artifacts?${this.tokenParam}`); + const artifacts = await response.json(); + const artifact = artifacts.find(item => item.path === artifactPath); + if (!artifact) { + throw new Error(`Missing artifact (${artifactPath}) for CircleCI build: ${buildNumber}`); + } + return artifact.url; + } catch (error) { + throw new Error(`CircleCI artifact URL request failed (${error.message})`); + } + } +} diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/constants.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/constants.ts index c5064c5dfc..186f75cae6 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/constants.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/constants.ts @@ -1,3 +1,4 @@ // Constants +export const AIO_DOWNLOADS_DIR = '/tmp/aio-downloads'; export const HIDDEN_DIR_PREFIX = 'hidden--'; export const SHORT_SHA_LEN = 7; diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/env-variables.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/env-variables.ts new file mode 100644 index 0000000000..bc55c90d9c --- /dev/null +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/env-variables.ts @@ -0,0 +1,19 @@ +import {getEnvVar} from './utils'; + +export const AIO_ARTIFACT_PATH = getEnvVar('AIO_ARTIFACT_PATH'); +export const AIO_BUILDS_DIR = getEnvVar('AIO_BUILDS_DIR'); +export const AIO_GITHUB_TOKEN = getEnvVar('AIO_GITHUB_TOKEN'); +export const AIO_CIRCLE_CI_TOKEN = getEnvVar('AIO_CIRCLE_CI_TOKEN'); +export const AIO_DOMAIN_NAME = getEnvVar('AIO_DOMAIN_NAME'); +export const AIO_GITHUB_ORGANIZATION = getEnvVar('AIO_GITHUB_ORGANIZATION'); +export const AIO_GITHUB_REPO = getEnvVar('AIO_GITHUB_REPO'); +export const AIO_GITHUB_TEAM_SLUGS = getEnvVar('AIO_GITHUB_TEAM_SLUGS'); +export const AIO_NGINX_HOSTNAME = getEnvVar('AIO_NGINX_HOSTNAME'); +export const AIO_NGINX_PORT_HTTP = +getEnvVar('AIO_NGINX_PORT_HTTP'); +export const AIO_NGINX_PORT_HTTPS = +getEnvVar('AIO_NGINX_PORT_HTTPS'); +export const AIO_SIGNIFICANT_FILES_PATTERN = getEnvVar('AIO_SIGNIFICANT_FILES_PATTERN'); +export const AIO_TRUSTED_PR_LABEL = getEnvVar('AIO_TRUSTED_PR_LABEL'); +export const AIO_UPLOAD_HOSTNAME = getEnvVar('AIO_UPLOAD_HOSTNAME'); +export const AIO_UPLOAD_PORT = +getEnvVar('AIO_UPLOAD_PORT'); +export const AIO_UPLOAD_MAX_SIZE = +getEnvVar('AIO_UPLOAD_MAX_SIZE'); +export const AIO_WWW_USER = getEnvVar('AIO_WWW_USER'); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/github-api.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/github-api.ts index 820944cb5f..62faba0d06 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/github-api.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/github-api.ts @@ -28,29 +28,17 @@ export class GithubApi { } // Methods - Public - public get(pathname: string, params?: RequestParamsOrNull): Promise { + public get(pathname: string, params?: RequestParamsOrNull): Promise { const path = this.buildPath(pathname, params); return this.request('get', path); } - public post(pathname: string, params?: RequestParamsOrNull, data?: any): Promise { + public post(pathname: string, params?: RequestParamsOrNull, data?: any): Promise { const path = this.buildPath(pathname, params); return this.request('post', path, data); } - // Methods - Protected - protected buildPath(pathname: string, params?: RequestParamsOrNull): string { - if (params == null) { - return pathname; - } - - const search = (params === null) ? '' : this.serializeSearchParams(params); - const joiner = search && '?'; - - return `${pathname}${joiner}${search}`; - } - - protected getPaginated(pathname: string, baseParams: RequestParams = {}, currentPage: number = 0): Promise { + public getPaginated(pathname: string, baseParams: RequestParams = {}, currentPage: number = 0): Promise { const perPage = 100; const params = { ...baseParams, @@ -67,7 +55,19 @@ export class GithubApi { }); } - protected request(method: string, path: string, data: any = null): Promise { + // Methods - Protected + protected buildPath(pathname: string, params?: RequestParamsOrNull) { + if (params == null) { + return pathname; + } + + const search = (params === null) ? '' : this.serializeSearchParams(params); + const joiner = search && '?'; + + return `${pathname}${joiner}${search}`; + } + + protected request(method: string, path: string, data: any = null) { return new Promise((resolve, reject) => { const options = { headers: {...this.requestHeaders}, @@ -81,7 +81,7 @@ export class GithubApi { reject(`Request to '${url}' failed (status: ${statusCode}): ${responseText}`); }; const onSuccess = (responseText: string) => { - try { resolve(JSON.parse(responseText)); } catch (err) { reject(err); } + try { resolve(responseText && JSON.parse(responseText)); } catch (err) { reject(err); } }; const onResponse = (res: IncomingMessage) => { const statusCode = res.statusCode || -1; @@ -101,7 +101,7 @@ export class GithubApi { }); } - protected serializeSearchParams(params: RequestParams): string { + protected serializeSearchParams(params: RequestParams) { return Object.keys(params). filter(key => params[key] != null). map(key => `${key}=${encodeURIComponent(String(params[key]))}`). diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/github-pull-requests.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/github-pull-requests.ts index 0075b17f98..eabd66d49a 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/github-pull-requests.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/github-pull-requests.ts @@ -1,46 +1,79 @@ -// Imports -import {assertNotMissingOrEmpty} from '../common/utils'; +import {assert, assertNotMissingOrEmpty} from '../common/utils'; import {GithubApi} from './github-api'; -// Interfaces - Types -export interface PullRequest { +export interface PullRequest { number: number; user: {login: string}; labels: {name: string}[]; } +export interface FileInfo { + sha: string; + filename: string; +} + export type PullRequestState = 'all' | 'closed' | 'open'; -// Classes -export class GithubPullRequests extends GithubApi { - // Constructor - constructor(githubToken: string, protected repoSlug: string) { - super(githubToken); - assertNotMissingOrEmpty('repoSlug', repoSlug); +/** + * Access pull requests on GitHub. + */ +export class GithubPullRequests { + public repoSlug: string; + + /** + * Create an instance of this helper + * @param api An instance of the Github API helper. + * @param githubOrg The organisation on GitHub whose repo we will interrogate. + * @param githubRepo The repository on Github with whose PRs we will interact. + */ + constructor(private api: GithubApi, githubOrg: string, githubRepo: string) { + assertNotMissingOrEmpty('githubOrg', githubOrg); + assertNotMissingOrEmpty('githubRepo', githubRepo); + this.repoSlug = `${githubOrg}/${githubRepo}`; } - // Methods - Public - public addComment(pr: number, body: string): Promise { - if (!(pr > 0)) { - throw new Error(`Invalid PR number: ${pr}`); - } else if (!body) { - throw new Error(`Invalid or empty comment body: ${body}`); - } - - return this.post(`/repos/${this.repoSlug}/issues/${pr}/comments`, null, {body}); + /** + * Post a comment on a PR. + * @param pr The number of the PR on which to comment. + * @param body The body of the comment to post. + * @returns A promise that resolves when the comment has been posted. + */ + public addComment(pr: number, body: string) { + assert(pr > 0, `Invalid PR number: ${pr}`); + assert(!!body, `Invalid or empty comment body: ${body}`); + return this.api.post(`/repos/${this.repoSlug}/issues/${pr}/comments`, null, {body}); } - public fetch(pr: number): Promise { + /** + * Request information about a PR. + * @param pr The number of the PR for which to request info. + * @returns A promise that is resolves with information about the specified PR. + */ + public fetch(pr: number) { + assert(pr > 0, `Invalid PR number: ${pr}`); // Using the `/issues/` URL, because the `/pulls/` one does not provide labels. - return this.get(`/repos/${this.repoSlug}/issues/${pr}`); + return this.api.get(`/repos/${this.repoSlug}/issues/${pr}`); } - public fetchAll(state: PullRequestState = 'all'): Promise { - console.log(`Fetching ${state} pull requests...`); - + /** + * Request information about all PRs that match the given state. + * @param state Only retrieve PRs that have this state. + * @returns A promise that is resolved with information about the requested PRs. + */ + public fetchAll(state: PullRequestState = 'all') { const pathname = `/repos/${this.repoSlug}/pulls`; const params = {state}; - return this.getPaginated(pathname, params); + return this.api.getPaginated(pathname, params); + } + + /** + * Request a list of files for the given PR. + * @param pr The number of the PR for which to request files. + * @returns A promise that resolves to an array of file information + */ + public fetchFiles(pr: number) { + assert(pr > 0, `Invalid PR number: ${pr}`); + return this.api.get(`/repos/${this.repoSlug}/pulls/${pr}/files`); } } diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/github-teams.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/github-teams.ts index ab1a2e0f16..fc882fdb8b 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/github-teams.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/github-teams.ts @@ -1,45 +1,72 @@ -// Imports import {assertNotMissingOrEmpty} from '../common/utils'; import {GithubApi} from './github-api'; -// Interfaces - Types -interface Team { +export interface Team { id: number; slug: string; } -interface TeamMembership { +export interface TeamMembership { state: string; } -// Classes -export class GithubTeams extends GithubApi { - // Constructor - constructor(githubToken: string, protected organization: string) { - super(githubToken); - assertNotMissingOrEmpty('organization', organization); +export class GithubTeams { + /** + * Create an instance of this helper + * @param api An instance of the Github API helper. + * @param githubOrg The organisation on GitHub whose repo we will interrogate. + */ + constructor(private api: GithubApi, protected githubOrg: string) { + assertNotMissingOrEmpty('githubOrg', githubOrg); } - // Methods - Public - public fetchAll(): Promise { - return this.getPaginated(`/orgs/${this.organization}/teams`); + /** + * Request information about all the organisation's teams in GitHub. + * @returns A promise that is resolved with information about the teams. + */ + public fetchAll() { + return this.api.getPaginated(`/orgs/${this.githubOrg}/teams`); } - public isMemberById(username: string, teamIds: number[]): Promise { - const getMembership = (teamId: number) => - this.get(`/teams/${teamId}/memberships/${username}`). - then(membership => membership.state === 'active'). - catch(() => false); - const reduceFn = (promise: Promise, teamId: number) => - promise.then(isMember => isMember || getMembership(teamId)); + /** + * Check whether the specified username is a member of the specified team. + * @param username The usernane to check for in the team. + * @param teamIds The team to check for the username. + * @returns a Promise that resolves to `true` if the username is a member of the team. + */ + public async isMemberById(username: string, teamIds: number[]) { - return teamIds.reduce(reduceFn, Promise.resolve(false)); + const getMembership = async (teamId: number) => { + try { + const {state} = await this.api.get(`/teams/${teamId}/memberships/${username}`); + return state === 'active'; + } catch (error) { + return false; + } + }; + + for (const teamId of teamIds) { + if (await getMembership(teamId)) { + return true; + } + } + + return false; } - public isMemberBySlug(username: string, teamSlugs: string[]): Promise { - return this.fetchAll(). - then(teams => teams.filter(team => teamSlugs.includes(team.slug)).map(team => team.id)). - then(teamIds => this.isMemberById(username, teamIds)). - catch(() => false); + /** + * Check whether the given username is a member of the teams specified by the team slugs. + * @param username The username to check for in the teams. + * @param teamSlugs A collection of slugs that represent the teams to check for the the username. + * @returns a Promise that resolves to `true` if the usernane is a member of at least one of the specified teams. + */ + public async isMemberBySlug(username: string, teamSlugs: string[]) { + try { + const teams = await this.fetchAll(); + const teamIds = teams.filter(team => teamSlugs.includes(team.slug)).map(team => team.id); + return await this.isMemberById(username, teamIds); + } catch (error) { + return false; + } } } diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/utils.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/utils.ts index 50e4ee5b7d..9855eee9fa 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/utils.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/common/utils.ts @@ -1,16 +1,74 @@ -// Functions -export const assertNotMissingOrEmpty = (name: string, value: string | null | undefined) => { +import {basename, resolve as resolvePath} from 'path'; +import {SHORT_SHA_LEN} from './constants'; + +/** + * Shorten a SHA to make it more readable + * @param sha The SHA to shorten. + */ +export function computeShortSha(sha: string) { + return sha.substr(0, SHORT_SHA_LEN); +} + +/** + * Compute the path for a downloaded artifact file. + * @param downloadsDir The directory where artifacts are downloaded + * @param pr The PR associated with this artifact. + * @param sha The SHA associated with the build for this artifact. + * @param artifactPath The path to the artifact on CircleCI. + * @returns The fully resolved location for the specified downloaded artifact. + */ +export function computeArtifactDownloadPath(downloadsDir: string, pr: number, sha: string, artifactPath: string) { + return resolvePath(downloadsDir, `${pr}-${computeShortSha(sha)}-${basename(artifactPath)}`); +} + +/** + * Extract the PR number and latest commit SHA from a downloaded file path. + * @param downloadPath the path to the downloaded file. + * @returns An object whose keys are the PR and SHA extracted from the file path. + */ +export function getPrInfoFromDownloadPath(downloadPath: string) { + const file = basename(downloadPath); + const [pr, sha] = file.split('-'); + return {pr: +pr, sha}; +} + +/** + * Assert that a value is true. + * @param value The value to assert. + * @param message The message if the value is not true. + */ +export function assert(value: boolean, message: string) { if (!value) { - throw new Error(`Missing or empty required parameter '${name}'!`); + throw new Error(message); } +} + +/** + * Assert that a parameter is not equal to "". + * @param name The name of the parameter. + * @param value The value of the parameter. + */ +export const assertNotMissingOrEmpty = (name: string, value: string | null | undefined) => { + assert(!!value, `Missing or empty required parameter '${name}'!`); }; +/** + * Get an environment variable. + * @param name The name of the environment variable. + * @param isOptional True if the variable is optional. + * @returns The value of the variable or "" if it is optional and falsy. + * @throws `Error` if the variable is falsy and not optional. + */ export const getEnvVar = (name: string, isOptional = false): string => { const value = process.env[name]; if (!isOptional && !value) { - console.error(`ERROR: Missing required environment variable '${name}'!`); - process.exit(1); + try { + throw new Error(`ERROR: Missing required environment variable '${name}'!`); + } catch (error) { + console.error(error.stack); + process.exit(1); + } } return value || ''; diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-creator.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-creator.ts index 899b3ab7be..65baf2c168 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-creator.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-creator.ts @@ -4,8 +4,8 @@ import {EventEmitter} from 'events'; import * as fs from 'fs'; import * as path from 'path'; import * as shell from 'shelljs'; -import {HIDDEN_DIR_PREFIX, SHORT_SHA_LEN} from '../common/constants'; -import {assertNotMissingOrEmpty} from '../common/utils'; +import {HIDDEN_DIR_PREFIX} from '../common/constants'; +import {assertNotMissingOrEmpty, computeShortSha} from '../common/utils'; import {ChangedPrVisibilityEvent, CreatedBuildEvent} from './build-events'; import {UploadError} from './upload-error'; @@ -18,9 +18,9 @@ export class BuildCreator extends EventEmitter { } // Methods - Public - public create(pr: string, sha: string, archivePath: string, isPublic: boolean): Promise { + public create(pr: number, sha: string, archivePath: string, isPublic: boolean): Promise { // Use only part of the SHA for more readable URLs. - sha = sha.substr(0, SHORT_SHA_LEN); + sha = computeShortSha(sha); const {newPrDir: prDir} = this.getCandidatePrDirs(pr, isPublic); const shaDir = path.join(prDir, sha); @@ -57,7 +57,7 @@ export class BuildCreator extends EventEmitter { }); } - public updatePrVisibility(pr: string, makePublic: boolean): Promise { + public updatePrVisibility(pr: number, makePublic: boolean): Promise { const {oldPrDir: otherVisPrDir, newPrDir: targetVisPrDir} = this.getCandidatePrDirs(pr, makePublic); return Promise. @@ -116,9 +116,9 @@ export class BuildCreator extends EventEmitter { }); } - protected getCandidatePrDirs(pr: string, isPublic: boolean) { + protected getCandidatePrDirs(pr: number, isPublic: boolean) { const hiddenPrDir = path.join(this.buildsDir, HIDDEN_DIR_PREFIX + pr); - const publicPrDir = path.join(this.buildsDir, pr); + const publicPrDir = path.join(this.buildsDir, `${pr}`); const oldPrDir = isPublic ? hiddenPrDir : publicPrDir; const newPrDir = isPublic ? publicPrDir : hiddenPrDir; diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-retriever.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-retriever.ts new file mode 100644 index 0000000000..11018e5e64 --- /dev/null +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-retriever.ts @@ -0,0 +1,83 @@ +import * as fs from 'fs'; +import fetch from 'node-fetch'; +import {dirname} from 'path'; +import {mkdir} from 'shelljs'; +import {promisify} from 'util'; +import {CircleCiApi} from '../common/circle-ci-api'; +import {assert, assertNotMissingOrEmpty, computeArtifactDownloadPath, createLogger} from '../common/utils'; +import {UploadError} from '../upload-server/upload-error'; + +export interface GithubInfo { + org: string; + pr: number; + repo: string; + sha: string; + success: boolean; +} + +/** + * A helper that can get information about builds and download build artifacts. + */ +export class BuildRetriever { + private logger = createLogger('BuildRetriever'); + constructor(private api: CircleCiApi, private downloadSizeLimit: number, private downloadDir: string) { + assert(downloadSizeLimit > 0, 'Invalid parameter "downloadSizeLimit" should be a number greater than 0.'); + assertNotMissingOrEmpty('downloadDir', downloadDir); + } + + /** + * Get GitHub information about a build + * @param buildNum The number of the build for which to retrieve the info. + * @returns The Github org, repo, PR and latest SHA for the specified build. + */ + public async getGithubInfo(buildNum: number) { + const buildInfo = await this.api.getBuildInfo(buildNum); + const githubInfo: GithubInfo = { + org: buildInfo.username, + pr: getPrfromBranch(buildInfo.branch), + repo: buildInfo.reponame, + sha: buildInfo.vcs_revision, + success: !buildInfo.failed, + }; + return githubInfo; + } + + /** + * Make a request to the given URL for a build artifact and store it locally. + * @param buildNum the number of the CircleCI build whose artifact we want to download. + * @param pr the number of the PR that triggered the CircleCI build. + * @param sha the commit in the PR that triggered the CircleCI build. + * @param artifactPath the path on CircleCI where the artifact was stored. + * @returns A promise to the file path where the downloaded file was stored. + */ + public async downloadBuildArtifact(buildNum: number, pr: number, sha: string, artifactPath: string) { + try { + const outPath = computeArtifactDownloadPath(this.downloadDir, pr, sha, artifactPath); + const downloadExists = await new Promise(resolve => fs.exists(outPath, exists => resolve(exists))); + if (!downloadExists) { + const url = await this.api.getBuildArtifactUrl(buildNum, artifactPath); + const response = await fetch(url, {size: this.downloadSizeLimit}); + if (response.status !== 200) { + throw new UploadError(response.status, `Error ${response.status} - ${response.statusText}`); + } + const buffer = await response.buffer(); + mkdir('-p', dirname(outPath)); + await promisify(fs.writeFile)(outPath, buffer); + } + return outPath; + } catch (error) { + this.logger.warn(error); + const status = (error.type === 'max-size') ? 413 : 500; + throw new UploadError(status, `CircleCI artifact download failed (${error.message || error})`); + } + } +} + +function getPrfromBranch(branch: string) { + // CircleCI only exposes PR numbers via the `branch` field :-( + const match = /^pull\/(\d+)$/.exec(branch); + if (!match) { + throw new Error(`No PR found in branch field: ${branch}`); + } + return +match[1]; +} diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-verifier.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-verifier.ts index bfabb47525..1281723e04 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-verifier.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/build-verifier.ts @@ -1,87 +1,46 @@ -// Imports -import * as jwt from 'jsonwebtoken'; import {GithubPullRequests, PullRequest} from '../common/github-pull-requests'; import {GithubTeams} from '../common/github-teams'; import {assertNotMissingOrEmpty} from '../common/utils'; -import {UploadError} from './upload-error'; -// Interfaces - Types -interface JwtPayload { - slug: string; - 'pull-request': number; -} - -// Enums -export enum BUILD_VERIFICATION_STATUS { - verifiedAndTrusted, - verifiedNotTrusted, -} - -// Classes +/** + * A helper to verify whether builds are trusted. + */ export class BuildVerifier { - // Properties - Protected - protected githubPullRequests: GithubPullRequests; - protected githubTeams: GithubTeams; - - // Constructor - constructor(protected secret: string, githubToken: string, protected repoSlug: string, organization: string, + /** + * Construct a new BuildVerifier instance. + * @param prs A helper to access PR information. + * @param teams A helper to access Github team information. + * @param allowedTeamSlugs The teams that are trusted. + * @param trustedPrLabel The github label that indicates that a PR is trusted. + */ + constructor(protected prs: GithubPullRequests, protected teams: GithubTeams, protected allowedTeamSlugs: string[], protected trustedPrLabel: string) { - assertNotMissingOrEmpty('secret', secret); - assertNotMissingOrEmpty('githubToken', githubToken); - assertNotMissingOrEmpty('repoSlug', repoSlug); - assertNotMissingOrEmpty('organization', organization); assertNotMissingOrEmpty('allowedTeamSlugs', allowedTeamSlugs && allowedTeamSlugs.join('')); assertNotMissingOrEmpty('trustedPrLabel', trustedPrLabel); - - this.githubPullRequests = new GithubPullRequests(githubToken, repoSlug); - this.githubTeams = new GithubTeams(githubToken, organization); } - // Methods - Public - public getPrIsTrusted(pr: number): Promise { - return Promise.resolve(). - then(() => this.githubPullRequests.fetch(pr)). - then(prInfo => this.hasLabel(prInfo, this.trustedPrLabel) || - this.githubTeams.isMemberBySlug(prInfo.user.login, this.allowedTeamSlugs)); + /** + * Check whether a PR contains files that are significant to the build. + * @param pr The number of the PR to check + * @param significantFilePattern A regex that selects files that are significant. + */ + public async getSignificantFilesChanged(pr: number, significantFilePattern: RegExp) { + const files = await this.prs.fetchFiles(pr); + return files.some(file => significantFilePattern.test(file.filename)); } - public verify(expectedPr: number, authHeader: string): Promise { - return Promise.resolve(). - then(() => this.extractJwtString(authHeader)). - then(jwtString => this.verifyJwt(expectedPr, jwtString)). - then(jwtPayload => this.verifyPr(jwtPayload['pull-request'])). - catch(err => { throw new UploadError(403, `Error while verifying upload for PR ${expectedPr}: ${err}`); }); - } - - // Methods - Protected - protected extractJwtString(input: string): string { - return input.replace(/^token +/i, ''); + /** + * Check whether a PR is trusted. + * @param pr The number of the PR to check. + * @returns true if the PR is trusted. + */ + public async getPrIsTrusted(pr: number): Promise { + const prInfo = await this.prs.fetch(pr); + return this.hasLabel(prInfo, this.trustedPrLabel) || + (await this.teams.isMemberBySlug(prInfo.user.login, this.allowedTeamSlugs)); } protected hasLabel(prInfo: PullRequest, label: string) { return prInfo.labels.some(labelObj => labelObj.name === label); } - - protected verifyJwt(expectedPr: number, token: string): Promise { - return new Promise((resolve, reject) => { - jwt.verify(token, this.secret, {issuer: 'Travis CI, GmbH'}, (err, payload: JwtPayload) => { - if (err) { - reject(err.message || err); - } else if (payload.slug !== this.repoSlug) { - reject(`jwt slug invalid. expected: ${this.repoSlug}`); - } else if (payload['pull-request'] !== expectedPr) { - reject(`jwt pull-request invalid. expected: ${expectedPr}`); - } else { - resolve(payload); - } - }); - }); - } - - protected verifyPr(pr: number): Promise { - return this.getPrIsTrusted(pr). - then(isTrusted => Promise.resolve(isTrusted ? - BUILD_VERIFICATION_STATUS.verifiedAndTrusted : - BUILD_VERIFICATION_STATUS.verifiedNotTrusted)); - } } diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/index-preverify-pr.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/index-preverify-pr.ts index 6cc058e61b..94d1462c6b 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/index-preverify-pr.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/index-preverify-pr.ts @@ -1,4 +1,7 @@ // Imports +import {GithubApi} from '../common/github-api'; +import {GithubPullRequests} from '../common/github-pull-requests'; +import {GithubTeams} from '../common/github-teams'; import {getEnvVar} from '../common/utils'; import {BuildVerifier} from './build-verifier'; @@ -7,16 +10,17 @@ _main(); // Functions function _main() { - const secret = 'unused'; const githubToken = getEnvVar('AIO_GITHUB_TOKEN'); - const repoSlug = getEnvVar('AIO_REPO_SLUG'); - const organization = getEnvVar('AIO_GITHUB_ORGANIZATION'); + const githubOrg = getEnvVar('AIO_GITHUB_ORGANIZATION'); + const githubRepo = getEnvVar('AIO_GITHUB_REPO'); const allowedTeamSlugs = getEnvVar('AIO_GITHUB_TEAM_SLUGS').split(','); const trustedPrLabel = getEnvVar('AIO_TRUSTED_PR_LABEL'); const pr = +getEnvVar('AIO_PREVERIFY_PR'); - const buildVerifier = new BuildVerifier(secret, githubToken, repoSlug, organization, allowedTeamSlugs, - trustedPrLabel); + const githubApi = new GithubApi(githubToken); + const prs = new GithubPullRequests(githubApi, githubOrg, githubRepo); + const teams = new GithubTeams(githubApi, githubOrg); + const buildVerifier = new BuildVerifier(prs, teams, allowedTeamSlugs, trustedPrLabel); // Exit codes: // - 0: The PR can be automatically trusted (i.e. author belongs to trusted team or PR has the "trusted PR" label). diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/index.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/index.ts index e7b705c0c2..9ecc5662ac 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/index.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/index.ts @@ -1,34 +1,41 @@ // Imports -import {getEnvVar} from '../common/utils'; -import {uploadServerFactory} from './upload-server-factory'; - -// Constants -const AIO_BUILDS_DIR = getEnvVar('AIO_BUILDS_DIR'); -const AIO_DOMAIN_NAME = getEnvVar('AIO_DOMAIN_NAME'); -const AIO_GITHUB_ORGANIZATION = getEnvVar('AIO_GITHUB_ORGANIZATION'); -const AIO_GITHUB_TEAM_SLUGS = getEnvVar('AIO_GITHUB_TEAM_SLUGS'); -const AIO_GITHUB_TOKEN = getEnvVar('AIO_GITHUB_TOKEN'); -const AIO_PREVIEW_DEPLOYMENT_TOKEN = getEnvVar('AIO_PREVIEW_DEPLOYMENT_TOKEN'); -const AIO_REPO_SLUG = getEnvVar('AIO_REPO_SLUG'); -const AIO_TRUSTED_PR_LABEL = getEnvVar('AIO_TRUSTED_PR_LABEL'); -const AIO_UPLOAD_HOSTNAME = getEnvVar('AIO_UPLOAD_HOSTNAME'); -const AIO_UPLOAD_PORT = +getEnvVar('AIO_UPLOAD_PORT'); +import {AIO_DOWNLOADS_DIR} from '../common/constants'; +import { + AIO_ARTIFACT_PATH, + AIO_BUILDS_DIR, + AIO_CIRCLE_CI_TOKEN, + AIO_DOMAIN_NAME, + AIO_GITHUB_ORGANIZATION, + AIO_GITHUB_REPO, + AIO_GITHUB_TEAM_SLUGS, + AIO_GITHUB_TOKEN, + AIO_SIGNIFICANT_FILES_PATTERN, + AIO_TRUSTED_PR_LABEL, + AIO_UPLOAD_HOSTNAME, + AIO_UPLOAD_MAX_SIZE, + AIO_UPLOAD_PORT, +} from '../common/env-variables'; +import {UploadServerFactory} from './upload-server-factory'; // Run _main(); // Functions function _main() { - uploadServerFactory. - create({ + UploadServerFactory + .create({ + buildArtifactPath: AIO_ARTIFACT_PATH, buildsDir: AIO_BUILDS_DIR, + circleCiToken: AIO_CIRCLE_CI_TOKEN, domainName: AIO_DOMAIN_NAME, - githubOrganization: AIO_GITHUB_ORGANIZATION, + downloadSizeLimit: AIO_UPLOAD_MAX_SIZE, + downloadsDir: AIO_DOWNLOADS_DIR, + githubOrg: AIO_GITHUB_ORGANIZATION, + githubRepo: AIO_GITHUB_REPO, githubTeamSlugs: AIO_GITHUB_TEAM_SLUGS.split(','), githubToken: AIO_GITHUB_TOKEN, - repoSlug: AIO_REPO_SLUG, - secret: AIO_PREVIEW_DEPLOYMENT_TOKEN, + significantFilesPattern: AIO_SIGNIFICANT_FILES_PATTERN, trustedPrLabel: AIO_TRUSTED_PR_LABEL, - }). - listen(AIO_UPLOAD_PORT, AIO_UPLOAD_HOSTNAME); + }) + .listen(AIO_UPLOAD_PORT, AIO_UPLOAD_HOSTNAME); } diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/upload-server-factory.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/upload-server-factory.ts index 48e60e3cc1..9eb8e12025 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/upload-server-factory.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/upload-server-factory.ts @@ -2,70 +2,168 @@ import * as bodyParser from 'body-parser'; import * as express from 'express'; import * as http from 'http'; +import {CircleCiApi} from '../common/circle-ci-api'; +import {GithubApi} from '../common/github-api'; import {GithubPullRequests} from '../common/github-pull-requests'; -import {assertNotMissingOrEmpty} from '../common/utils'; +import {GithubTeams} from '../common/github-teams'; +import {assert, assertNotMissingOrEmpty, createLogger} from '../common/utils'; import {BuildCreator} from './build-creator'; import {ChangedPrVisibilityEvent, CreatedBuildEvent} from './build-events'; -import {BUILD_VERIFICATION_STATUS, BuildVerifier} from './build-verifier'; -import {UploadError} from './upload-error'; +import {BuildRetriever} from './build-retriever'; +import {BuildVerifier} from './build-verifier'; +import {respondWithError, throwRequestError} from './utils'; -// Constants -const AUTHORIZATION_HEADER = 'AUTHORIZATION'; -const X_FILE_HEADER = 'X-FILE'; +const AIO_PREVIEW_JOB = 'aio_preview'; // Interfaces - Types -interface UploadServerConfig { +export interface UploadServerConfig { + downloadsDir: string; + downloadSizeLimit: number; + buildArtifactPath: string; buildsDir: string; domainName: string; - githubOrganization: string; + githubOrg: string; + githubRepo: string; githubTeamSlugs: string[]; + circleCiToken: string; githubToken: string; - repoSlug: string; - secret: string; + significantFilesPattern: string; trustedPrLabel: string; } +const logger = createLogger('UploadServer'); + // Classes -class UploadServerFactory { +export class UploadServerFactory { // Methods - Public - public create({ - buildsDir, - domainName, - githubOrganization, - githubTeamSlugs, - githubToken, - repoSlug, - secret, - trustedPrLabel, - }: UploadServerConfig): http.Server { - assertNotMissingOrEmpty('domainName', domainName); + public static create(cfg: UploadServerConfig): http.Server { + assertNotMissingOrEmpty('domainName', cfg.domainName); - const buildVerifier = new BuildVerifier(secret, githubToken, repoSlug, githubOrganization, githubTeamSlugs, - trustedPrLabel); - const buildCreator = this.createBuildCreator(buildsDir, githubToken, repoSlug, domainName); + const circleCiApi = new CircleCiApi(cfg.githubOrg, cfg.githubRepo, cfg.circleCiToken); + const githubApi = new GithubApi(cfg.githubToken); + const prs = new GithubPullRequests(githubApi, cfg.githubOrg, cfg.githubRepo); + const teams = new GithubTeams(githubApi, cfg.githubOrg); - const middleware = this.createMiddleware(buildVerifier, buildCreator); + const buildRetriever = new BuildRetriever(circleCiApi, cfg.downloadSizeLimit, cfg.downloadsDir); + const buildVerifier = new BuildVerifier(prs, teams, cfg.githubTeamSlugs, cfg.trustedPrLabel); + const buildCreator = UploadServerFactory.createBuildCreator(prs, cfg.buildsDir, cfg.domainName); + + const middleware = UploadServerFactory.createMiddleware(buildRetriever, buildVerifier, buildCreator, cfg); const httpServer = http.createServer(middleware as any); httpServer.on('listening', () => { const info = httpServer.address(); - console.info(`Up and running (and listening on ${info.address}:${info.port})...`); + logger.info(`Up and running (and listening on ${info.address}:${info.port})...`); }); return httpServer; } - // Methods - Protected - protected createBuildCreator(buildsDir: string, githubToken: string, repoSlug: string, - domainName: string): BuildCreator { + public static createMiddleware(buildRetriever: BuildRetriever, buildVerifier: BuildVerifier, + buildCreator: BuildCreator, cfg: UploadServerConfig): express.Express { + const middleware = express(); + const jsonParser = bodyParser.json(); + + // RESPOND TO IS-ALIVE PING + middleware.get(/^\/health-check\/?$/, (_req, res) => res.sendStatus(200)); + + // CIRCLE_CI BUILD COMPLETE WEBHOOK + middleware.post(/^\/circle-build\/?$/, jsonParser, async (req, res) => { + try { + if (!( + req.is('json') && + req.body && + req.body.payload && + req.body.payload.build_num > 0 && + req.body.payload.build_parameters && + req.body.payload.build_parameters.CIRCLE_JOB + )) { + throwRequestError(400, `Incorrect body content. Expected JSON`, req); + } + + const job = req.body.payload.build_parameters.CIRCLE_JOB; + const buildNum = req.body.payload.build_num; + + logger.log(`Build:${buildNum}, Job:${job} - processing web-hook trigger`); + + if (job !== AIO_PREVIEW_JOB) { + res.sendStatus(204); + logger.log(`Build:${buildNum}, Job:${job} -`, + `Skipping preview processing because this is not the "${AIO_PREVIEW_JOB}" job.`); + return; + } + + const { pr, sha, org, repo, success } = await buildRetriever.getGithubInfo(buildNum); + + if (!success) { + res.sendStatus(204); + logger.log(`PR:${pr}, Build:${buildNum} - Skipping preview processing because this build did not succeed.`); + return; + } + + assert(cfg.githubOrg === org, + `Invalid webhook: expected "githubOrg" property to equal "${cfg.githubOrg}" but got "${org}".`); + assert(cfg.githubRepo === repo, + `Invalid webhook: expected "githubRepo" property to equal "${cfg.githubRepo}" but got "${repo}".`); + + // Do not deploy unless this PR has touched relevant files: `aio/` or `packages/` (except for spec files) + if (!await buildVerifier.getSignificantFilesChanged(pr, new RegExp(cfg.significantFilesPattern))) { + res.sendStatus(204); + logger.log(`PR:${pr}, Build:${buildNum} - ` + + `Skipping preview processing because this PR did not touch any significant files.`); + return; + } + + const artifactPath = await buildRetriever.downloadBuildArtifact(buildNum, pr, sha, cfg.buildArtifactPath); + const isPublic = await buildVerifier.getPrIsTrusted(pr); + await buildCreator.create(pr, sha, artifactPath, isPublic); + res.sendStatus(isPublic ? 201 : 202); + } catch (err) { + logger.error('CircleCI webhook error', err); + respondWithError(res, err); + } + }); + + // GITHUB PR UPDATED WEBHOOK + middleware.post(/^\/pr-updated\/?$/, jsonParser, async (req, res) => { + const { action, number: prNo }: { action?: string, number?: number } = req.body; + const visMayHaveChanged = !action || (action === 'labeled') || (action === 'unlabeled'); + + try { + if (!visMayHaveChanged) { + res.sendStatus(200); + } else if (!prNo) { + throwRequestError(400, `Missing or empty 'number' field`, req); + } else { + const isPublic = await buildVerifier.getPrIsTrusted(prNo); + await buildCreator.updatePrVisibility(prNo, isPublic); + res.sendStatus(200); + } + } catch (err) { + logger.error('PR update hook error', err); + respondWithError(res, err); + } + }); + + // ALL OTHER REQUESTS + middleware.all('*', req => throwRequestError(404, 'Unknown resource', req)); + middleware.use((err: any, _req: any, res: express.Response, _next: any) => { + const statusText = http.STATUS_CODES[err.status] || '???'; + logger.error(`Upload error: ${err.status} - ${statusText}:`, err.message); + respondWithError(res, err); + }); + + return middleware; + } + + public static createBuildCreator(prs: GithubPullRequests, buildsDir: string, domainName: string) { const buildCreator = new BuildCreator(buildsDir); - const githubPullRequests = new GithubPullRequests(githubToken, repoSlug); const postPreviewsComment = (pr: number, shas: string[]) => { const body = shas. map(sha => `You can preview ${sha} at https://pr${pr}-${sha}.${domainName}/.`). join('\n'); - return githubPullRequests.addComment(pr, body); + return prs.addComment(pr, body); }; buildCreator.on(CreatedBuildEvent.type, ({pr, sha, isPublic}: CreatedBuildEvent) => { @@ -82,72 +180,4 @@ class UploadServerFactory { return buildCreator; } - - protected createMiddleware(buildVerifier: BuildVerifier, buildCreator: BuildCreator): express.Express { - const middleware = express(); - const jsonParser = bodyParser.json(); - - middleware.get(/^\/create-build\/([1-9][0-9]*)\/([0-9a-f]{40})\/?$/, (req, res) => { - const pr = req.params[0]; - const sha = req.params[1]; - const archive = req.header(X_FILE_HEADER); - const authHeader = req.header(AUTHORIZATION_HEADER); - - if (!authHeader) { - this.throwRequestError(401, `Missing or empty '${AUTHORIZATION_HEADER}' header`, req); - } else if (!archive) { - this.throwRequestError(400, `Missing or empty '${X_FILE_HEADER}' header`, req); - } else { - Promise.resolve(). - then(() => buildVerifier.verify(+pr, authHeader)). - then(verStatus => verStatus === BUILD_VERIFICATION_STATUS.verifiedAndTrusted). - then(isPublic => buildCreator.create(pr, sha, archive, isPublic). - then(() => res.sendStatus(isPublic ? 201 : 202))). - catch(err => this.respondWithError(res, err)); - } - }); - middleware.get(/^\/health-check\/?$/, (_req, res) => res.sendStatus(200)); - middleware.post(/^\/pr-updated\/?$/, jsonParser, (req, res) => { - const {action, number: prNo}: {action?: string, number?: number} = req.body; - const visMayHaveChanged = !action || (action === 'labeled') || (action === 'unlabeled'); - - if (!visMayHaveChanged) { - res.sendStatus(200); - } else if (!prNo) { - this.throwRequestError(400, `Missing or empty 'number' field`, req); - } else { - Promise.resolve(). - then(() => buildVerifier.getPrIsTrusted(prNo)). - then(isPublic => buildCreator.updatePrVisibility(String(prNo), isPublic)). - then(() => res.sendStatus(200)). - catch(err => this.respondWithError(res, err)); - } - }); - middleware.all('*', req => this.throwRequestError(404, 'Unknown resource', req)); - middleware.use((err: any, _req: any, res: express.Response, _next: any) => this.respondWithError(res, err)); - - return middleware; - } - - protected respondWithError(res: express.Response, err: any) { - if (!(err instanceof UploadError)) { - err = new UploadError(500, String((err && err.message) || err)); - } - - const statusText = http.STATUS_CODES[err.status] || '???'; - console.error(`Upload error: ${err.status} - ${statusText}`); - console.error(err.message); - - res.status(err.status).end(err.message); - } - - protected throwRequestError(status: number, error: string, req: express.Request) { - const message = `${error} in request: ${req.method} ${req.originalUrl}` + - (!req.body ? '' : ` ${JSON.stringify(req.body)}`); - - throw new UploadError(status, message); - } } - -// Exports -export const uploadServerFactory = new UploadServerFactory(); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/utils.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/utils.ts new file mode 100644 index 0000000000..1bd1d99cb1 --- /dev/null +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/upload-server/utils.ts @@ -0,0 +1,34 @@ +import * as express from 'express'; +import * as http from 'http'; +import {promisify} from 'util'; +import {UploadError} from './upload-error'; + +/** + * Update the response to report that an error has occurred. + * @param res The response to configure as an error. + * @param err The error that needs to be reported. + */ +export async function respondWithError(res: express.Response, err: any) { + if (!(err instanceof UploadError)) { + err = new UploadError(500, String((err && err.message) || err)); + } + + const statusText = http.STATUS_CODES[err.status] || '???'; + console.error(`Upload error: ${err.status} - ${statusText}`); + console.error(err.message); + + res.status(err.status); + await promisify(res.end.bind(res))(err.message); +} + +/** + * Throw an exception that describes the given error information. + * @param status The HTTP status code include in the error. + * @param error The error message to include in the error. + * @param req The request that triggered this error. + */ +export function throwRequestError(status: number, error: string, req: express.Request): never { + const message = `${error} in request: ${req.method} ${req.originalUrl}` + + (!req.body ? '' : ` ${JSON.stringify(req.body)}`); + throw new UploadError(status, message); +} diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/constants.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/constants.ts index 92e7f15a31..19ce093879 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/constants.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/constants.ts @@ -1,16 +1,37 @@ -// Using the values below, we can fake the response of the corresponding methods in tests. This is -// necessary, because the test upload-server will be running as a separate node process, so we will -// not have direct access to the code (e.g. for mocking). -// (See also 'lib/verify-setup/start-test-upload-server.ts'.) +export const enum BuildNums { + BUILD_INFO_ERROR = 1, + BUILD_INFO_404, + BUILD_INFO_BUILD_FAILED, + BUILD_INFO_INVALID_GH_ORG, + BUILD_INFO_INVALID_GH_REPO, + CHANGED_FILES_ERROR, + CHANGED_FILES_404, + CHANGED_FILES_NONE, + BUILD_ARTIFACTS_ERROR, + BUILD_ARTIFACTS_404, + BUILD_ARTIFACTS_EMPTY, + BUILD_ARTIFACTS_MISSING, + DOWNLOAD_ARTIFACT_ERROR, + DOWNLOAD_ARTIFACT_404, + DOWNLOAD_ARTIFACT_TOO_BIG, + TRUST_CHECK_ERROR, + TRUST_CHECK_UNTRUSTED, + TRUST_CHECK_TRUSTED_LABEL, + TRUST_CHECK_ACTIVE_TRUSTED_USER, + TRUST_CHECK_INACTIVE_TRUSTED_USER, +} -/* tslint:disable: variable-name */ +export const enum PrNums { + CHANGED_FILES_ERROR = 1, + CHANGED_FILES_404, + CHANGED_FILES_NONE, + TRUST_CHECK_ERROR, + TRUST_CHECK_UNTRUSTED, + TRUST_CHECK_TRUSTED_LABEL, + TRUST_CHECK_ACTIVE_TRUSTED_USER, + TRUST_CHECK_INACTIVE_TRUSTED_USER, +} -// Special values to be used as `authHeader` in `BuildVerifier#verify()`. -export const BV_verify_error = 'FAKE_VERIFICATION_ERROR'; -export const BV_verify_verifiedNotTrusted = 'FAKE_VERIFIED_NOT_TRUSTED'; - -// Special values to be used as `pr` in `BuildVerifier#getPrIsTrusted()`. -export const BV_getPrIsTrusted_error = 32203; -export const BV_getPrIsTrusted_notTrusted = 72457; - -/* tslint:enable: variable-name */ +export const SHA = '1234567890'.repeat(4); +export const ALT_SHA = 'abcde'.repeat(8); +export const SIMILAR_SHA = SHA.slice(0, -1) + 'A'; diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/delete-empty.d.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/delete-empty.d.ts new file mode 100644 index 0000000000..19b96d7255 --- /dev/null +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/delete-empty.d.ts @@ -0,0 +1,10 @@ +declare module 'delete-empty' { + interface Options { + dryRun: boolean; + verbose: boolean; + filter: (filePath: string) => boolean; + } + export default function deleteEmpty(cwd: string, options?: Options): Promise; + export default function deleteEmpty(cwd: string, options?: Options, callback?: (err: any, deleted: string[]) => void): void; + export function sync(cwd: string, options?: Options): string[]; +} diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/helper.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/helper.ts index e5b2322494..ae9e8be174 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/helper.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/helper.ts @@ -4,18 +4,14 @@ import * as fs from 'fs'; import * as http from 'http'; import * as path from 'path'; import * as shell from 'shelljs'; -import {HIDDEN_DIR_PREFIX, SHORT_SHA_LEN} from '../common/constants'; -import {getEnvVar} from '../common/utils'; - -// Constans -const TEST_AIO_BUILDS_DIR = getEnvVar('TEST_AIO_BUILDS_DIR'); -const TEST_AIO_NGINX_HOSTNAME = getEnvVar('TEST_AIO_NGINX_HOSTNAME'); -const TEST_AIO_NGINX_PORT_HTTP = +getEnvVar('TEST_AIO_NGINX_PORT_HTTP'); -const TEST_AIO_NGINX_PORT_HTTPS = +getEnvVar('TEST_AIO_NGINX_PORT_HTTPS'); -const TEST_AIO_UPLOAD_HOSTNAME = getEnvVar('TEST_AIO_UPLOAD_HOSTNAME'); -const TEST_AIO_UPLOAD_MAX_SIZE = +getEnvVar('TEST_AIO_UPLOAD_MAX_SIZE'); -const TEST_AIO_UPLOAD_PORT = +getEnvVar('TEST_AIO_UPLOAD_PORT'); -const WWW_USER = getEnvVar('AIO_WWW_USER'); +import {AIO_DOWNLOADS_DIR, HIDDEN_DIR_PREFIX} from '../common/constants'; +import { + AIO_BUILDS_DIR, + AIO_NGINX_PORT_HTTP, + AIO_NGINX_PORT_HTTPS, + AIO_WWW_USER, +} from '../common/env-variables'; +import {computeShortSha} from '../common/utils'; // Interfaces - Types export interface CmdResult { success: boolean; err: Error | null; stdout: string; stderr: string; } @@ -27,61 +23,47 @@ export type VerifyCmdResultFn = (result: CmdResult) => void; // Classes class Helper { - // Properties - Public - public get buildsDir() { return TEST_AIO_BUILDS_DIR; } - public get nginxHostname() { return TEST_AIO_NGINX_HOSTNAME; } - public get nginxPortHttp() { return TEST_AIO_NGINX_PORT_HTTP; } - public get nginxPortHttps() { return TEST_AIO_NGINX_PORT_HTTPS; } - public get uploadHostname() { return TEST_AIO_UPLOAD_HOSTNAME; } - public get uploadPort() { return TEST_AIO_UPLOAD_PORT; } - public get uploadMaxSize() { return TEST_AIO_UPLOAD_MAX_SIZE; } - public get wwwUser() { return WWW_USER; } - // Properties - Protected protected cleanUpFns: CleanUpFn[] = []; protected portPerScheme: {[scheme: string]: number} = { - http: this.nginxPortHttp, - https: this.nginxPortHttps, + http: AIO_NGINX_PORT_HTTP, + https: AIO_NGINX_PORT_HTTPS, }; // Constructor constructor() { - shell.mkdir('-p', this.buildsDir); - shell.exec(`chown -R ${this.wwwUser} ${this.buildsDir}`); + shell.mkdir('-p', AIO_BUILDS_DIR); + shell.exec(`chown -R ${AIO_WWW_USER} ${AIO_BUILDS_DIR}`); + shell.mkdir('-p', AIO_DOWNLOADS_DIR); + shell.exec(`chown -R ${AIO_WWW_USER} ${AIO_DOWNLOADS_DIR}`); } // Methods - Public - public buildExists(pr: string, sha = '', isPublic = true, legacy = false): boolean { - const prDir = this.getPrDir(pr, isPublic); - const dir = !sha ? prDir : this.getShaDir(prDir, sha, legacy); - return fs.existsSync(dir); - } - public cleanUp() { while (this.cleanUpFns.length) { // Clean-up fns remove themselves from the list. this.cleanUpFns[0](); } - if (fs.readdirSync(this.buildsDir).length) { - throw new Error(`Directory '${this.buildsDir}' is not empty after clean-up.`); + const leftoverDownloads = fs.readdirSync(AIO_DOWNLOADS_DIR); + const leftoverBuilds = fs.readdirSync(AIO_BUILDS_DIR); + + if (leftoverDownloads.length) { + console.log(`Downloads directory '${AIO_DOWNLOADS_DIR}' is not empty after clean-up.`, leftoverDownloads); + shell.rm('-rf', `${AIO_DOWNLOADS_DIR}/*`); + } + + if (leftoverBuilds.length) { + console.log(`Builds directory '${AIO_BUILDS_DIR}' is not empty after clean-up.`, leftoverBuilds); + shell.rm('-rf', `${AIO_BUILDS_DIR}/*`); + } + + if (leftoverBuilds.length || leftoverDownloads.length) { + throw new Error(`Unexpected test files not cleaned up.`); } } - public createDummyArchive(pr: string, sha: string, archivePath: string): CleanUpFn { - const inputDir = this.getShaDir(this.getPrDir(`uploaded/${pr}`, true), sha); - const cmd1 = `tar --create --gzip --directory "${inputDir}" --file "${archivePath}" .`; - const cmd2 = `chown ${this.wwwUser} ${archivePath}`; - - const cleanUpTemp = this.createDummyBuild(`uploaded/${pr}`, sha, true, true); - shell.exec(cmd1); - shell.exec(cmd2); - cleanUpTemp(); - - return this.createCleanUpFn(() => shell.rm('-rf', archivePath)); - } - - public createDummyBuild(pr: string, sha: string, isPublic = true, force = false, legacy = false): CleanUpFn { + public createDummyBuild(pr: number, sha: string, isPublic = true, force = false, legacy = false) { const prDir = this.getPrDir(pr, isPublic); const shaDir = this.getShaDir(prDir, sha, legacy); const idxPath = path.join(shaDir, 'index.html'); @@ -89,34 +71,21 @@ class Helper { this.writeFile(idxPath, {content: `PR: ${pr} | SHA: ${sha} | File: /index.html`}, force); this.writeFile(barPath, {content: `PR: ${pr} | SHA: ${sha} | File: /foo/bar.js`}, force); - shell.exec(`chown -R ${this.wwwUser} ${prDir}`); + shell.exec(`chown -R ${AIO_WWW_USER} ${prDir}`); return this.createCleanUpFn(() => shell.rm('-rf', prDir)); } - public deletePrDir(pr: string, isPublic = true) { - const prDir = this.getPrDir(pr, isPublic); - - if (fs.existsSync(prDir)) { - shell.chmod('-R', 'a+w', prDir); - shell.rm('-rf', prDir); - } - } - - public getPrDir(pr: string, isPublic: boolean): string { - const prDirName = isPublic ? pr : HIDDEN_DIR_PREFIX + pr; - return path.join(this.buildsDir, prDirName); + public getPrDir(pr: number, isPublic: boolean): string { + const prDirName = isPublic ? '' + pr : HIDDEN_DIR_PREFIX + pr; + return path.join(AIO_BUILDS_DIR, prDirName); } public getShaDir(prDir: string, sha: string, legacy = false): string { - return path.join(prDir, legacy ? sha : this.getShordSha(sha)); + return path.join(prDir, legacy ? sha : computeShortSha(sha)); } - public getShordSha(sha: string): string { - return sha.substr(0, SHORT_SHA_LEN); - } - - public readBuildFile(pr: string, sha: string, relFilePath: string, isPublic = true, legacy = false): string { + public readBuildFile(pr: number, sha: string, relFilePath: string, isPublic = true, legacy = false): string { const shaDir = this.getShaDir(this.getPrDir(pr, isPublic), sha, legacy); const absFilePath = path.join(shaDir, relFilePath); return fs.readFileSync(absFilePath, 'utf8'); @@ -164,14 +133,14 @@ class Helper { }; } - public writeBuildFile(pr: string, sha: string, relFilePath: string, content: string, isPublic = true, - legacy = false): CleanUpFn { + public writeBuildFile(pr: number, sha: string, relFilePath: string, content: string, isPublic = true, + legacy = false) { const shaDir = this.getShaDir(this.getPrDir(pr, isPublic), sha, legacy); const absFilePath = path.join(shaDir, relFilePath); - return this.writeFile(absFilePath, {content}, true); + this.writeFile(absFilePath, {content}, true); } - public writeFile(filePath: string, {content, size}: FileSpecs, force = false): CleanUpFn { + public writeFile(filePath: string, {content, size}: FileSpecs, force = false) { if (!force && fs.existsSync(filePath)) { throw new Error(`Refusing to overwrite existing file '${filePath}'.`); } @@ -189,9 +158,7 @@ class Helper { // Create a file with the specified content. fs.writeFileSync(filePath, content || ''); } - shell.exec(`chown ${this.wwwUser} ${filePath}`); - - return this.createCleanUpFn(() => shell.rm('-rf', cleanUpTarget)); + shell.exec(`chown ${AIO_WWW_USER} ${filePath}`); } // Methods - Protected @@ -210,5 +177,43 @@ class Helper { } } +interface CurlOptions { + method?: string; + options?: string; + data?: any; + url?: string; + extraPath?: string; +} + +export function makeCurl(baseUrl: string) { + return function curl({ + method = 'POST', + options = '', + data = {}, + url = baseUrl, + extraPath = '', + }: CurlOptions) { + const dataString = data ? JSON.stringify(data) : ''; + const cmd = `curl -iLX ${method} ` + + `${options} ` + + `--header "Content-Type: application/json" ` + + `--data '${dataString}' ` + + `${url}${extraPath}`; + return helper.runCmd(cmd); + }; +} + +export function payload(buildNum: number) { + return { + data: { + payload: { + build_num: buildNum, + build_parameters: { CIRCLE_JOB: 'aio_preview' }, + }, + }, + }; +} + + // Exports export const helper = new Helper(); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/jasmine-custom-matchers-types.d.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/jasmine-custom-matchers-types.d.ts new file mode 100644 index 0000000000..8788325666 --- /dev/null +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/jasmine-custom-matchers-types.d.ts @@ -0,0 +1,7 @@ +declare module jasmine { + interface Matchers { + toExistAsAFile(remove = true): boolean; + toExistAsABuild(remove = true): boolean; + toExistAsAnArtifact(remove = true): boolean; + } +} \ No newline at end of file diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/jasmine-custom-matchers.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/jasmine-custom-matchers.ts new file mode 100644 index 0000000000..48e0190fbc --- /dev/null +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/jasmine-custom-matchers.ts @@ -0,0 +1,86 @@ +import {sync as deleteEmpty} from 'delete-empty'; +import {existsSync, unlinkSync} from 'fs'; +import {join} from 'path'; +import {AIO_DOWNLOADS_DIR} from '../common/constants'; +import {computeShortSha} from '../common/utils'; +import {SHA} from './constants'; +import {helper} from './helper'; + +function checkFile(filePath: string, remove: boolean) { + const exists = existsSync(filePath); + if (exists && remove) { + // if we expected the file to exist then we remove it to prevent leftover file errors + unlinkSync(filePath); + } + return exists; +} + +function getArtifactPath(prNum: number, sha: string = SHA) { + return `${AIO_DOWNLOADS_DIR}/${prNum}-${computeShortSha(sha)}-aio-snapshot.tgz`; +} + +function checkFiles(prNum: number, isPublic: boolean, sha: string, isLegacy: boolean, remove: boolean) { + const files = ['/index.html', '/foo/bar.js']; + const prPath = helper.getPrDir(prNum, isPublic); + const shaPath = helper.getShaDir(prPath, sha, isLegacy); + + const existingFiles: string[] = []; + const missingFiles: string[] = []; + files + .map(file => join(shaPath, file)) + .forEach(file => (checkFile(file, remove) ? existingFiles : missingFiles).push(file)); + + deleteEmpty(prPath); + + return { existingFiles, missingFiles }; +} + +class ToExistAsAFile { + public compare(actual: string, remove = true) { + const pass = checkFile(actual, remove); + return { + message: `Expected file at "${actual}" ${pass ? 'not' : ''} to exist`, + pass, + }; + } +} + +class ToExistAsAnArtifact { + public compare(actual: {prNum: number, sha?: string}, remove = true) { + const { prNum, sha = SHA } = actual; + const filePath = getArtifactPath(prNum, sha); + const pass = checkFile(filePath, remove); + return { + message: `Expected artifact "PR:${prNum}, SHA:${sha}, FILE:${filePath}" ${pass ? 'not' : '\b'} to exist`, + pass, + }; + } +} + +class ToExistAsABuild { + public compare(actual: {prNum: number, isPublic?: boolean, sha?: string, isLegacy?: boolean}, remove = true) { + const {prNum, isPublic = true, sha = SHA, isLegacy = false} = actual; + const {missingFiles} = checkFiles(prNum, isPublic, sha, isLegacy, remove); + return { + message: `Expected files for build "PR:${prNum}, SHA:${sha}" to exist:\n` + + missingFiles.map(file => ` - ${file}`).join('\n'), + pass: missingFiles.length === 0, + }; + } + public negativeCompare(actual: {prNum: number, isPublic?: boolean, sha?: string, isLegacy?: boolean}) { + const {prNum, isPublic = true, sha = SHA, isLegacy = false} = actual; + const { existingFiles } = checkFiles(prNum, isPublic, sha, isLegacy, false); + return { + message: `Expected files for build "PR:${prNum}, SHA:${sha}" not to exist:\n` + + existingFiles.map(file => ` - ${file}`).join('\n'), + pass: existingFiles.length === 0, + }; + } + +} + +export const customMatchers = { + toExistAsABuild: () => new ToExistAsABuild(), + toExistAsAFile: () => new ToExistAsAFile(), + toExistAsAnArtifact: () => new ToExistAsAnArtifact(), +}; diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/mock-external-apis.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/mock-external-apis.ts new file mode 100644 index 0000000000..a311de881b --- /dev/null +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/mock-external-apis.ts @@ -0,0 +1,170 @@ +/* tslint:disable:max-line-length */ +import * as nock from 'nock'; +import * as tar from 'tar-stream'; +import {gzipSync} from 'zlib'; +import {getEnvVar} from '../common/utils'; +import {BuildNums, PrNums, SHA} from './constants'; + +// We are using the `nock` library to fake responses from REST requests, when testing. +// This is necessary, because the test upload-server runs as a separate node process to +// the test harness, so we do not have direct access to the code (e.g. for mocking). +// (See also 'lib/verify-setup/start-test-upload-server.ts'.) + +// Each of the potential requests to an external API (e.g. Github or CircleCI) are mocked +// below and return a suitable response. This is quite complicated to setup since the +// response from, say, CircleCI will affect what request is made to, say, Github. + +const log = (...args: any[]) => { + // Filter out non-matching URL checks + if (!/^matching.+: false$/.test(args[0])) { + args.unshift('>> NOCK:'); + console.log.apply(console, args); + } +}; + +const AIO_CIRCLE_CI_TOKEN = getEnvVar('AIO_CIRCLE_CI_TOKEN'); +const AIO_GITHUB_TOKEN = getEnvVar('AIO_GITHUB_TOKEN'); + +const AIO_ARTIFACT_PATH = getEnvVar('AIO_ARTIFACT_PATH'); +const AIO_GITHUB_ORGANIZATION = getEnvVar('AIO_GITHUB_ORGANIZATION'); +const AIO_GITHUB_REPO = getEnvVar('AIO_GITHUB_REPO'); +const AIO_TRUSTED_PR_LABEL = getEnvVar('AIO_TRUSTED_PR_LABEL'); +const AIO_GITHUB_TEAM_SLUGS = getEnvVar('AIO_GITHUB_TEAM_SLUGS').split(','); + +const ACTIVE_TRUSTED_USER = 'active-trusted-user'; +const INACTIVE_TRUSTED_USER = 'inactive-trusted-user'; +const UNTRUSTED_USER = 'untrusted-user'; + +const BASIC_BUILD_INFO = { + branch: `pull/${PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER}`, + failed: false, + reponame: AIO_GITHUB_REPO, + username: AIO_GITHUB_ORGANIZATION, + vcs_revision: SHA, +}; + +const ISSUE_INFO_TRUSTED_LABEL = { labels: [{ name: AIO_TRUSTED_PR_LABEL }], user: { login: UNTRUSTED_USER } }; +const ISSUE_INFO_ACTIVE_TRUSTED_USER = { labels: [], user: { login: ACTIVE_TRUSTED_USER } }; +const ISSUE_INFO_INACTIVE_TRUSTED_USER = { labels: [], user: { login: INACTIVE_TRUSTED_USER } }; +const ISSUE_INFO_UNTRUSTED = { labels: [], user: { login: UNTRUSTED_USER } }; +const ACTIVE_STATE = { state: 'active' }; +const INACTIVE_STATE = { state: 'inactive' }; + +const TEST_TEAM_INFO = AIO_GITHUB_TEAM_SLUGS.map((slug, index) => ({ slug, id: index })); + +const CIRCLE_CI_API_HOST = 'https://circleci.com'; +const CIRCLE_CI_TOKEN_PARAM = `circle-token=${AIO_CIRCLE_CI_TOKEN}`; +const ARTIFACT_1 = { path: 'artifact-1', url: `${CIRCLE_CI_API_HOST}/artifacts/artifact-1`, _urlPath: '/artifacts/artifact-1' }; +const ARTIFACT_2 = { path: 'artifact-2', url: `${CIRCLE_CI_API_HOST}/artifacts/artifact-2`, _urlPath: '/artifacts/artifact-2' }; +const ARTIFACT_3 = { path: 'artifact-3', url: `${CIRCLE_CI_API_HOST}/artifacts/artifact-3`, _urlPath: '/artifacts/artifact-3' }; +const ARTIFACT_ERROR = { path: AIO_ARTIFACT_PATH, url: `${CIRCLE_CI_API_HOST}/artifacts/error`, _urlPath: '/artifacts/error' }; +const ARTIFACT_404 = { path: AIO_ARTIFACT_PATH, url: `${CIRCLE_CI_API_HOST}/artifacts/404`, _urlPath: '/artifacts/404' }; +const ARTIFACT_VALID_TRUSTED_USER = { path: AIO_ARTIFACT_PATH, url: `${CIRCLE_CI_API_HOST}/artifacts/valid/user`, _urlPath: '/artifacts/valid/user' }; +const ARTIFACT_VALID_TRUSTED_LABEL = { path: AIO_ARTIFACT_PATH, url: `${CIRCLE_CI_API_HOST}/artifacts/valid/label`, _urlPath: '/artifacts/valid/label' }; +const ARTIFACT_VALID_UNTRUSTED = { path: AIO_ARTIFACT_PATH, url: `${CIRCLE_CI_API_HOST}/artifacts/valid/untrusted`, _urlPath: '/artifacts/valid/untrusted' }; + +const CIRCLE_CI_BUILD_INFO_URL = `/api/v1.1/project/github/${AIO_GITHUB_ORGANIZATION}/${AIO_GITHUB_REPO}`; + +const buildInfoUrl = (buildNum: number) => `${CIRCLE_CI_BUILD_INFO_URL}/${buildNum}?${CIRCLE_CI_TOKEN_PARAM}`; +const buildArtifactsUrl = (buildNum: number) => `${CIRCLE_CI_BUILD_INFO_URL}/${buildNum}/artifacts?${CIRCLE_CI_TOKEN_PARAM}`; +const buildInfo = (prNum: number) => ({ ...BASIC_BUILD_INFO, branch: `pull/${prNum}` }); + +const GITHUB_API_HOST = 'https://api.github.com'; +const GITHUB_ISSUES_URL = `/repos/${AIO_GITHUB_ORGANIZATION}/${AIO_GITHUB_REPO}/issues`; +const GITHUB_PULLS_URL = `/repos/${AIO_GITHUB_ORGANIZATION}/${AIO_GITHUB_REPO}/pulls`; +const GITHUB_TEAMS_URL = `/orgs/${AIO_GITHUB_ORGANIZATION}/teams`; + +const getIssueUrl = (prNum: number) => `${GITHUB_ISSUES_URL}/${prNum}`; +const getFilesUrl = (prNum: number) => `${GITHUB_PULLS_URL}/${prNum}/files`; +const getCommentUrl = (prNum: number) => `${getIssueUrl(prNum)}/comments`; +const getTeamMembershipUrl = (teamId: number, username: string) => `/teams/${teamId}/memberships/${username}`; + +const createArchive = (buildNum: number, prNum: number, sha: string) => { + console.log('createArchive', buildNum, prNum, sha); + const pack = tar.pack(); + pack.entry({name: 'index.html'}, `BUILD: ${buildNum} | PR: ${prNum} | SHA: ${sha} | File: /index.html`); + pack.entry({name: 'foo/bar.js'}, `BUILD: ${buildNum} | PR: ${prNum} | SHA: ${sha} | File: /foo/bar.js`); + pack.finalize(); + const zip = gzipSync(pack.read()); + return zip; +}; + +// Create request scopes +const circleCiApi = nock(CIRCLE_CI_API_HOST).log(log).persist(); +const githubApi = nock(GITHUB_API_HOST).log(log).persist().matchHeader('Authorization', `token ${AIO_GITHUB_TOKEN}`); + +////////////////////////////// + +// GENERAL responses +githubApi.get(GITHUB_TEAMS_URL + '?page=0&per_page=100').reply(200, TEST_TEAM_INFO); +githubApi.post(getCommentUrl(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER)).reply(200); + +// BUILD_INFO errors +circleCiApi.get(buildInfoUrl(BuildNums.BUILD_INFO_ERROR)).replyWithError('BUILD_INFO_ERROR'); +circleCiApi.get(buildInfoUrl(BuildNums.BUILD_INFO_404)).reply(404, 'BUILD_INFO_404'); +circleCiApi.get(buildInfoUrl(BuildNums.BUILD_INFO_BUILD_FAILED)).reply(200, { ...BASIC_BUILD_INFO, failed: true }); +circleCiApi.get(buildInfoUrl(BuildNums.BUILD_INFO_INVALID_GH_ORG)).reply(200, { ...BASIC_BUILD_INFO, username: 'bad' }); +circleCiApi.get(buildInfoUrl(BuildNums.BUILD_INFO_INVALID_GH_REPO)).reply(200, { ...BASIC_BUILD_INFO, reponame: 'bad' }); + +// CHANGED FILE errors +circleCiApi.get(buildInfoUrl(BuildNums.CHANGED_FILES_ERROR)).reply(200, buildInfo(PrNums.CHANGED_FILES_ERROR)); +githubApi.get(getFilesUrl(PrNums.CHANGED_FILES_ERROR)).replyWithError('CHANGED_FILES_ERROR'); +circleCiApi.get(buildInfoUrl(BuildNums.CHANGED_FILES_404)).reply(200, buildInfo(PrNums.CHANGED_FILES_404)); +githubApi.get(getFilesUrl(PrNums.CHANGED_FILES_404)).reply(404, 'CHANGED_FILES_404'); +circleCiApi.get(buildInfoUrl(BuildNums.CHANGED_FILES_NONE)).reply(200, buildInfo(PrNums.CHANGED_FILES_NONE)); +githubApi.get(getFilesUrl(PrNums.CHANGED_FILES_NONE)).reply(200, []); + +// ARTIFACT URL errors +circleCiApi.get(buildInfoUrl(BuildNums.BUILD_ARTIFACTS_ERROR)).reply(200, buildInfo(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER)); +circleCiApi.get(buildArtifactsUrl(BuildNums.BUILD_ARTIFACTS_ERROR)).replyWithError('BUILD_ARTIFACTS_ERROR'); +circleCiApi.get(buildInfoUrl(BuildNums.BUILD_ARTIFACTS_404)).reply(200, buildInfo(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER)); +circleCiApi.get(buildArtifactsUrl(BuildNums.BUILD_ARTIFACTS_404)).reply(404, 'BUILD_ARTIFACTS_ERROR'); +circleCiApi.get(buildInfoUrl(BuildNums.BUILD_ARTIFACTS_EMPTY)).reply(200, buildInfo(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER)); +circleCiApi.get(buildArtifactsUrl(BuildNums.BUILD_ARTIFACTS_EMPTY)).reply(200, []); +circleCiApi.get(buildInfoUrl(BuildNums.BUILD_ARTIFACTS_MISSING)).reply(200, buildInfo(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER)); +circleCiApi.get(buildArtifactsUrl(BuildNums.BUILD_ARTIFACTS_MISSING)).reply(200, [ARTIFACT_1, ARTIFACT_2, ARTIFACT_3]); + +// ARTIFACT DOWNLOAD errors +circleCiApi.get(buildInfoUrl(BuildNums.DOWNLOAD_ARTIFACT_ERROR)).reply(200, buildInfo(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER)); +circleCiApi.get(buildArtifactsUrl(BuildNums.DOWNLOAD_ARTIFACT_ERROR)).reply(200, [ARTIFACT_ERROR]); +circleCiApi.get(ARTIFACT_ERROR._urlPath).replyWithError(ARTIFACT_ERROR._urlPath); +circleCiApi.get(buildInfoUrl(BuildNums.DOWNLOAD_ARTIFACT_404)).reply(200, buildInfo(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER)); +circleCiApi.get(buildArtifactsUrl(BuildNums.DOWNLOAD_ARTIFACT_404)).reply(200, [ARTIFACT_404]); +circleCiApi.get(ARTIFACT_ERROR._urlPath).reply(404, ARTIFACT_ERROR._urlPath); + +// TRUST CHECK errors +circleCiApi.get(buildInfoUrl(BuildNums.TRUST_CHECK_ERROR)).reply(200, buildInfo(PrNums.TRUST_CHECK_ERROR)); +githubApi.get(getFilesUrl(PrNums.TRUST_CHECK_ERROR)).reply(200, [{ filename: 'aio/a' }]); +circleCiApi.get(buildArtifactsUrl(BuildNums.TRUST_CHECK_ERROR)).reply(200, [ARTIFACT_VALID_TRUSTED_USER]); +githubApi.get(getIssueUrl(PrNums.TRUST_CHECK_ERROR)).replyWithError('TRUST_CHECK_ERROR'); + +// ACTIVE TRUSTED USER response +circleCiApi.get(buildInfoUrl(BuildNums.TRUST_CHECK_ACTIVE_TRUSTED_USER)).reply(200, BASIC_BUILD_INFO); +githubApi.get(getFilesUrl(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER)).reply(200, [{ filename: 'aio/a' }]); +circleCiApi.get(buildArtifactsUrl(BuildNums.TRUST_CHECK_ACTIVE_TRUSTED_USER)).reply(200, [ARTIFACT_VALID_TRUSTED_USER]); +circleCiApi.get(ARTIFACT_VALID_TRUSTED_USER._urlPath).reply(200, createArchive(BuildNums.TRUST_CHECK_ACTIVE_TRUSTED_USER, PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER, SHA)); +githubApi.get(getIssueUrl(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER)).reply(200, ISSUE_INFO_ACTIVE_TRUSTED_USER); +githubApi.get(getTeamMembershipUrl(0, ACTIVE_TRUSTED_USER)).reply(200, ACTIVE_STATE); + +// TRUSTED LABEL response +circleCiApi.get(buildInfoUrl(BuildNums.TRUST_CHECK_TRUSTED_LABEL)).reply(200, BASIC_BUILD_INFO); +githubApi.get(getFilesUrl(PrNums.TRUST_CHECK_TRUSTED_LABEL)).reply(200, [{ filename: 'aio/a' }]); +circleCiApi.get(buildArtifactsUrl(BuildNums.TRUST_CHECK_TRUSTED_LABEL)).reply(200, [ARTIFACT_VALID_TRUSTED_LABEL]); +circleCiApi.get(ARTIFACT_VALID_TRUSTED_LABEL._urlPath).reply(200, createArchive(BuildNums.TRUST_CHECK_TRUSTED_LABEL, PrNums.TRUST_CHECK_TRUSTED_LABEL, SHA)); +githubApi.get(getIssueUrl(PrNums.TRUST_CHECK_TRUSTED_LABEL)).reply(200, ISSUE_INFO_TRUSTED_LABEL); +githubApi.get(getTeamMembershipUrl(0, ACTIVE_TRUSTED_USER)).reply(200, ACTIVE_STATE); + +// INACTIVE TRUSTED USER response +circleCiApi.get(buildInfoUrl(BuildNums.TRUST_CHECK_INACTIVE_TRUSTED_USER)).reply(200, BASIC_BUILD_INFO); +githubApi.get(getFilesUrl(PrNums.TRUST_CHECK_INACTIVE_TRUSTED_USER)).reply(200, [{ filename: 'aio/a' }]); +circleCiApi.get(buildArtifactsUrl(BuildNums.TRUST_CHECK_INACTIVE_TRUSTED_USER)).reply(200, [ARTIFACT_VALID_TRUSTED_USER]); +githubApi.get(getIssueUrl(PrNums.TRUST_CHECK_INACTIVE_TRUSTED_USER)).reply(200, ISSUE_INFO_INACTIVE_TRUSTED_USER); +githubApi.get(getTeamMembershipUrl(0, INACTIVE_TRUSTED_USER)).reply(200, INACTIVE_STATE); + +// UNTRUSTED reponse +circleCiApi.get(buildInfoUrl(BuildNums.TRUST_CHECK_UNTRUSTED)).reply(200, buildInfo(PrNums.TRUST_CHECK_UNTRUSTED)); +githubApi.get(getFilesUrl(PrNums.TRUST_CHECK_UNTRUSTED)).reply(200, [{ filename: 'aio/a' }]); +circleCiApi.get(buildArtifactsUrl(BuildNums.TRUST_CHECK_UNTRUSTED)).reply(200, [ARTIFACT_VALID_UNTRUSTED]); +circleCiApi.get(ARTIFACT_VALID_UNTRUSTED._urlPath).reply(200, createArchive(BuildNums.TRUST_CHECK_UNTRUSTED, PrNums.TRUST_CHECK_UNTRUSTED, SHA)); +githubApi.get(getIssueUrl(PrNums.TRUST_CHECK_UNTRUSTED)).reply(200, ISSUE_INFO_UNTRUSTED); +githubApi.get(getTeamMembershipUrl(0, UNTRUSTED_USER)).reply(404); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/nginx.e2e.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/nginx.e2e.ts index a3e15992f3..3dd639d470 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/nginx.e2e.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/nginx.e2e.ts @@ -1,17 +1,22 @@ // Imports import * as path from 'path'; +import {rm} from 'shelljs'; +import {AIO_BUILDS_DIR, AIO_NGINX_HOSTNAME, AIO_NGINX_PORT_HTTP, AIO_NGINX_PORT_HTTPS} from '../common/env-variables'; +import {computeShortSha} from '../common/utils'; import {helper as h} from './helper'; +import {customMatchers} from './jasmine-custom-matchers'; // Tests describe(`nginx`, () => { - beforeEach(() => jasmine.DEFAULT_TIMEOUT_INTERVAL = 10000); + beforeEach(() => jasmine.DEFAULT_TIMEOUT_INTERVAL = 5000); + beforeEach(() => jasmine.addMatchers(customMatchers)); afterEach(() => h.cleanUp()); it('should redirect HTTP to HTTPS', done => { - const httpHost = `${h.nginxHostname}:${h.nginxPortHttp}`; - const httpsHost = `${h.nginxHostname}:${h.nginxPortHttps}`; + const httpHost = `${AIO_NGINX_HOSTNAME}:${AIO_NGINX_PORT_HTTP}`; + const httpsHost = `${AIO_NGINX_HOSTNAME}:${AIO_NGINX_PORT_HTTPS}`; const urlMap = { [`http://${httpHost}/`]: `https://${httpsHost}/`, [`http://${httpHost}/foo`]: `https://${httpsHost}/foo`, @@ -32,13 +37,13 @@ describe(`nginx`, () => { h.runForAllSupportedSchemes((scheme, port) => describe(`(on ${scheme.toUpperCase()})`, () => { - const hostname = h.nginxHostname; + const hostname = AIO_NGINX_HOSTNAME; const host = `${hostname}:${port}`; - const pr = '9'; + const pr = 9; const sha9 = '9'.repeat(40); const sha0 = '0'.repeat(40); - const shortSha9 = h.getShordSha(sha9); - const shortSha0 = h.getShordSha(sha0); + const shortSha9 = computeShortSha(sha9); + const shortSha0 = computeShortSha(sha0); describe(`pr-.${host}/*`, () => { @@ -50,6 +55,11 @@ describe(`nginx`, () => { h.createDummyBuild(pr, sha0); }); + afterEach(() => { + expect({ prNum: pr, sha: sha9 }).toExistAsABuild(); + expect({ prNum: pr, sha: sha0 }).toExistAsABuild(); + }); + it('should return /index.html', done => { const origin = `${scheme}://pr${pr}-${shortSha9}.${host}`; @@ -63,17 +73,19 @@ describe(`nginx`, () => { }); - it('should return /index.html (for legacy builds)', done => { + it('should return /index.html (for legacy builds)', async () => { const origin = `${scheme}://pr${pr}-${sha9}.${host}`; const bodyRegex = new RegExp(`^PR: ${pr} | SHA: ${sha9} | File: /index\\.html$`); h.createDummyBuild(pr, sha9, true, false, true); - Promise.all([ + await Promise.all([ h.runCmd(`curl -iL ${origin}/index.html`).then(h.verifyResponse(200, bodyRegex)), h.runCmd(`curl -iL ${origin}/`).then(h.verifyResponse(200, bodyRegex)), h.runCmd(`curl -iL ${origin}`).then(h.verifyResponse(200, bodyRegex)), - ]).then(done); + ]); + + expect({ prNum: pr, sha: sha9, isLegacy: true }).toExistAsABuild(); }); @@ -86,15 +98,15 @@ describe(`nginx`, () => { }); - it('should return /foo/bar.js (for legacy builds)', done => { + it('should return /foo/bar.js (for legacy builds)', async () => { const origin = `${scheme}://pr${pr}-${sha9}.${host}`; const bodyRegex = new RegExp(`^PR: ${pr} | SHA: ${sha9} | File: /foo/bar\\.js$`); h.createDummyBuild(pr, sha9, true, false, true); - h.runCmd(`curl -iL ${origin}/foo/bar.js`). - then(h.verifyResponse(200, bodyRegex)). - then(done); + await h.runCmd(`curl -iL ${origin}/foo/bar.js`).then(h.verifyResponse(200, bodyRegex)); + + expect({ prNum: pr, sha: sha9, isLegacy: true }).toExistAsABuild(); }); @@ -126,7 +138,7 @@ describe(`nginx`, () => { it('should respond with 404 for unknown PRs/SHAs', done => { const otherPr = 54321; - const otherShortSha = h.getShordSha('8'.repeat(40)); + const otherShortSha = computeShortSha('8'.repeat(40)); Promise.all([ h.runCmd(`curl -iL ${scheme}://pr${pr}9-${shortSha9}.${host}`).then(h.verifyResponse(404)), @@ -174,39 +186,41 @@ describe(`nginx`, () => { describe('(for hidden builds)', () => { - it('should respond with 404 for any file or directory', done => { + it('should respond with 404 for any file or directory', async () => { const origin = `${scheme}://pr${pr}-${shortSha9}.${host}`; const assert404 = h.verifyResponse(404); h.createDummyBuild(pr, sha9, false); - expect(h.buildExists(pr, sha9, false)).toBe(true); - Promise.all([ + await Promise.all([ h.runCmd(`curl -iL ${origin}/index.html`).then(assert404), h.runCmd(`curl -iL ${origin}/`).then(assert404), h.runCmd(`curl -iL ${origin}`).then(assert404), h.runCmd(`curl -iL ${origin}/foo/bar.js`).then(assert404), h.runCmd(`curl -iL ${origin}/foo/`).then(assert404), h.runCmd(`curl -iL ${origin}/foo`).then(assert404), - ]).then(done); + ]); + + expect({ prNum: pr, sha: sha9, isPublic: false }).toExistAsABuild(); }); - it('should respond with 404 for any file or directory (for legacy builds)', done => { + it('should respond with 404 for any file or directory (for legacy builds)', async () => { const origin = `${scheme}://pr${pr}-${sha9}.${host}`; const assert404 = h.verifyResponse(404); h.createDummyBuild(pr, sha9, false, false, true); - expect(h.buildExists(pr, sha9, false, true)).toBe(true); - Promise.all([ + await Promise.all([ h.runCmd(`curl -iL ${origin}/index.html`).then(assert404), h.runCmd(`curl -iL ${origin}/`).then(assert404), h.runCmd(`curl -iL ${origin}`).then(assert404), h.runCmd(`curl -iL ${origin}/foo/bar.js`).then(assert404), h.runCmd(`curl -iL ${origin}/foo/`).then(assert404), h.runCmd(`curl -iL ${origin}/foo`).then(assert404), - ]).then(done); + ]); + + expect({ prNum: pr, sha: sha9, isPublic: false, isLegacy: true }).toExistAsABuild(); }); }); @@ -238,10 +252,10 @@ describe(`nginx`, () => { }); - describe(`${host}/create-build//`, () => { + describe(`${host}/circle-build`, () => { it('should disallow non-POST requests', done => { - const url = `${scheme}://${host}/create-build/${pr}/${sha9}`; + const url = `${scheme}://${host}/circle-build`; Promise.all([ h.runCmd(`curl -iLX GET ${url}`).then(h.verifyResponse([405, 'Not Allowed'])), @@ -252,31 +266,9 @@ describe(`nginx`, () => { }); - it(`should reject files larger than ${h.uploadMaxSize}B (according to header)`, done => { - const headers = `--header "Content-Length: ${1.5 * h.uploadMaxSize}"`; - const url = `${scheme}://${host}/create-build/${pr}/${sha9}`; - - h.runCmd(`curl -iLX POST ${headers} ${url}`). - then(h.verifyResponse([413, 'Request Entity Too Large'])). - then(done); - }); - - - it(`should reject files larger than ${h.uploadMaxSize}B (without header)`, done => { - const filePath = path.join(h.buildsDir, 'snapshot.tar.gz'); - const url = `${scheme}://${host}/create-build/${pr}/${sha9}`; - - h.writeFile(filePath, {size: 1.5 * h.uploadMaxSize}); - - h.runCmd(`curl -iLX POST --data-binary "@${filePath}" ${url}`). - then(h.verifyResponse([413, 'Request Entity Too Large'])). - then(done); - }); - - it('should pass requests through to the upload server', done => { - h.runCmd(`curl -iLX POST ${scheme}://${host}/create-build/${pr}/${sha9}`). - then(h.verifyResponse(401, /Missing or empty 'AUTHORIZATION' header/)). + h.runCmd(`curl -iLX POST ${scheme}://${host}/circle-build`). + then(h.verifyResponse(400, /Incorrect body content. Expected JSON/)). then(done); }); @@ -285,35 +277,16 @@ describe(`nginx`, () => { const cmdPrefix = `curl -iLX POST ${scheme}://${host}`; Promise.all([ - h.runCmd(`${cmdPrefix}/foo/create-build/${pr}/${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/foo-create-build/${pr}/${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/fooncreate-build/${pr}/${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/create-build/foo/${pr}/${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/create-build-foo/${pr}/${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/create-buildnfoo/${pr}/${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/create-build/pr${pr}/${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/create-build/${pr}/${sha9}42`).then(h.verifyResponse(404)), + h.runCmd(`${cmdPrefix}/foo/circle-build/`).then(h.verifyResponse(404)), + h.runCmd(`${cmdPrefix}/foo-circle-build/`).then(h.verifyResponse(404)), + h.runCmd(`${cmdPrefix}/fooncircle-build/`).then(h.verifyResponse(404)), + h.runCmd(`${cmdPrefix}/circle-build/foo/`).then(h.verifyResponse(404)), + h.runCmd(`${cmdPrefix}/circle-build-foo/`).then(h.verifyResponse(404)), + h.runCmd(`${cmdPrefix}/circle-buildnfoo/`).then(h.verifyResponse(404)), + h.runCmd(`${cmdPrefix}/circle-build/pr`).then(h.verifyResponse(404)), + h.runCmd(`${cmdPrefix}/circle-build/42`).then(h.verifyResponse(404)), ]).then(done); }); - - - it('should reject PRs with leading zeros', done => { - h.runCmd(`curl -iLX POST ${scheme}://${host}/create-build/0${pr}/${sha9}`). - then(h.verifyResponse(404)). - then(done); - }); - - - it('should accept SHAs with leading zeros (but not trim the zeros)', done => { - const cmdPrefix = `curl -iLX POST ${scheme}://${host}/create-build/${pr}`; - const bodyRegex = /Missing or empty 'AUTHORIZATION' header/; - - Promise.all([ - h.runCmd(`${cmdPrefix}/0${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/${sha0}`).then(h.verifyResponse(401, bodyRegex)), - ]).then(done); - }); - }); @@ -335,13 +308,9 @@ describe(`nginx`, () => { const cmdPrefix = `curl -iLX POST --header "Content-Type: application/json"`; const cmd1 = `${cmdPrefix} ${url}`; - const cmd2 = `${cmdPrefix} --data '{"number":${pr}}' ${url}`; - const cmd3 = `${cmdPrefix} --data '{"number":${pr},"action":"foo"}' ${url}`; Promise.all([ h.runCmd(cmd1).then(h.verifyResponse(400, /Missing or empty 'number' field/)), - h.runCmd(cmd2).then(h.verifyResponse(200)), - h.runCmd(cmd3).then(h.verifyResponse(200)), ]).then(done); }); @@ -364,13 +333,15 @@ describe(`nginx`, () => { describe(`${host}/*`, () => { - it('should respond with 404 for unknown URLs (even if the resource exists)', done => { + beforeEach(() => { ['index.html', 'foo.js', 'foo/index.html'].forEach(relFilePath => { - const absFilePath = path.join(h.buildsDir, relFilePath); - h.writeFile(absFilePath, {content: `File: /${relFilePath}`}); + const absFilePath = path.join(AIO_BUILDS_DIR, relFilePath); + return h.writeFile(absFilePath, {content: `File: /${relFilePath}`}); }); + }); - Promise.all([ + it('should respond with 404 for unknown URLs (even if the resource exists)', async () => { + await Promise.all([ h.runCmd(`curl -iL ${scheme}://${host}/index.html`).then(h.verifyResponse(404)), h.runCmd(`curl -iL ${scheme}://${host}/`).then(h.verifyResponse(404)), h.runCmd(`curl -iL ${scheme}://${host}`).then(h.verifyResponse(404)), @@ -379,7 +350,14 @@ describe(`nginx`, () => { h.runCmd(`curl -iL ${scheme}://foo.${host}`).then(h.verifyResponse(404)), h.runCmd(`curl -iL ${scheme}://${host}/foo.js`).then(h.verifyResponse(404)), h.runCmd(`curl -iL ${scheme}://${host}/foo/index.html`).then(h.verifyResponse(404)), - ]).then(done); + ]); + }); + + afterEach(() => { + ['index.html', 'foo.js', 'foo/index.html', 'foo'].forEach(relFilePath => { + const absFilePath = path.join(AIO_BUILDS_DIR, relFilePath); + rm('-r', absFilePath); + }); }); }); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/server-integration.e2e.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/server-integration.e2e.ts index 6e9467e837..13caad776d 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/server-integration.e2e.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/server-integration.e2e.ts @@ -1,101 +1,80 @@ // Imports -import * as path from 'path'; -import * as c from './constants'; -import {helper as h} from './helper'; +import {AIO_NGINX_HOSTNAME} from '../common/env-variables'; +import {computeShortSha} from '../common/utils'; +import {ALT_SHA, BuildNums, PrNums, SHA} from './constants'; +import {helper as h, makeCurl, payload} from './helper'; +import {customMatchers} from './jasmine-custom-matchers'; // Tests h.runForAllSupportedSchemes((scheme, port) => describe(`integration (on ${scheme.toUpperCase()})`, () => { - const hostname = h.nginxHostname; + const hostname = AIO_NGINX_HOSTNAME; const host = `${hostname}:${port}`; - const pr9 = '9'; - const sha9 = '9'.repeat(40); - const sha0 = '0'.repeat(40); - const archivePath = path.join(h.buildsDir, 'snapshot.tar.gz'); + const curlPrUpdated = makeCurl(`${scheme}://${host}/pr-updated`); - const getFile = (pr: string, sha: string, file: string) => - h.runCmd(`curl -iL ${scheme}://pr${pr}-${h.getShordSha(sha)}.${host}/${file}`); - const uploadBuild = (pr: string, sha: string, archive: string, authHeader = 'Token FOO') => { - const curlPost = `curl -iLX POST --header "Authorization: ${authHeader}"`; - return h.runCmd(`${curlPost} --data-binary "@${archive}" ${scheme}://${host}/create-build/${pr}/${sha}`); - }; - const prUpdated = (pr: number, action?: string) => { - const url = `${scheme}://${host}/pr-updated`; - const payloadStr = JSON.stringify({number: pr, action}); - return h.runCmd(`curl -iLX POST --header "Content-Type: application/json" --data '${payloadStr}' ${url}`); - }; + const getFile = (pr: number, sha: string, file: string) => + h.runCmd(`curl -iL ${scheme}://pr${pr}-${computeShortSha(sha)}.${host}/${file}`); + const prUpdated = (prNum: number, action?: string) => curlPrUpdated({ data: { number: prNum, action } }); + const circleBuild = makeCurl(`${scheme}://${host}/circle-build`); - beforeEach(() => jasmine.DEFAULT_TIMEOUT_INTERVAL = 10000); - afterEach(() => { - h.deletePrDir(pr9); - h.deletePrDir(pr9, false); - h.cleanUp(); + beforeEach(() => { + jasmine.DEFAULT_TIMEOUT_INTERVAL = 5000; + jasmine.addMatchers(customMatchers); }); + afterEach(() => h.cleanUp()); describe('for a new/non-existing PR', () => { - it('should be able to upload and serve a public build', done => { - const regexPrefix9 = `^PR: uploaded\\/${pr9} \\| SHA: ${sha9} \\| File:`; - const idxContentRegex9 = new RegExp(`${regexPrefix9} \\/index\\.html$`); - const barContentRegex9 = new RegExp(`${regexPrefix9} \\/foo\\/bar\\.js$`); + it('should be able to upload and serve a public build', async () => { + const BUILD = BuildNums.TRUST_CHECK_ACTIVE_TRUSTED_USER; + const PR = PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER; - h.createDummyArchive(pr9, sha9, archivePath); + const regexPrefix = `^BUILD: ${BUILD} \\| PR: ${PR} \\| SHA: ${SHA} \\| File:`; + const idxContentRegex = new RegExp(`${regexPrefix} \\/index\\.html$`); + const barContentRegex = new RegExp(`${regexPrefix} \\/foo\\/bar\\.js$`); - uploadBuild(pr9, sha9, archivePath). - then(() => Promise.all([ - getFile(pr9, sha9, 'index.html').then(h.verifyResponse(200, idxContentRegex9)), - getFile(pr9, sha9, 'foo/bar.js').then(h.verifyResponse(200, barContentRegex9)), - ])). - then(done); + await circleBuild(payload(BUILD)).then(h.verifyResponse(201)); + await Promise.all([ + getFile(PR, SHA, 'index.html').then(h.verifyResponse(200, idxContentRegex)), + getFile(PR, SHA, 'foo/bar.js').then(h.verifyResponse(200, barContentRegex)), + ]); + + expect({ prNum: PR }).toExistAsABuild(); + expect({ prNum: PR, isPublic: false }).not.toExistAsABuild(); }); - it('should be able to upload but not serve a hidden build', done => { - const regexPrefix9 = `^PR: uploaded\\/${pr9} \\| SHA: ${sha9} \\| File:`; - const idxContentRegex9 = new RegExp(`${regexPrefix9} \\/index\\.html$`); - const barContentRegex9 = new RegExp(`${regexPrefix9} \\/foo\\/bar\\.js$`); + it('should be able to upload but not serve a hidden build', async () => { + const BUILD = BuildNums.TRUST_CHECK_UNTRUSTED; + const PR = PrNums.TRUST_CHECK_UNTRUSTED; - h.createDummyArchive(pr9, sha9, archivePath); + await circleBuild(payload(BUILD)).then(h.verifyResponse(202)); + await Promise.all([ + getFile(PR, SHA, 'index.html').then(h.verifyResponse(404)), + getFile(PR, SHA, 'foo/bar.js').then(h.verifyResponse(404)), + ]); - uploadBuild(pr9, sha9, archivePath, c.BV_verify_verifiedNotTrusted). - then(() => Promise.all([ - getFile(pr9, sha9, 'index.html').then(h.verifyResponse(404)), - getFile(pr9, sha9, 'foo/bar.js').then(h.verifyResponse(404)), - ])). - then(() => { - expect(h.buildExists(pr9, sha9)).toBe(false); - expect(h.buildExists(pr9, sha9, false)).toBe(true); - expect(h.readBuildFile(pr9, sha9, 'index.html', false)).toMatch(idxContentRegex9); - expect(h.readBuildFile(pr9, sha9, 'foo/bar.js', false)).toMatch(barContentRegex9); - }). - then(done); + expect({ prNum: PR }).not.toExistAsABuild(); + expect({ prNum: PR, isPublic: false }).toExistAsABuild(); }); - it('should reject an upload if verification fails', done => { - const errorRegex9 = new RegExp(`Error while verifying upload for PR ${pr9}: Test`); + it('should reject an upload if verification fails', async () => { + const BUILD = BuildNums.TRUST_CHECK_ERROR; + const PR = PrNums.TRUST_CHECK_ERROR; - h.createDummyArchive(pr9, sha9, archivePath); - - uploadBuild(pr9, sha9, archivePath, c.BV_verify_error). - then(h.verifyResponse(403, errorRegex9)). - then(() => { - expect(h.buildExists(pr9)).toBe(false); - expect(h.buildExists(pr9, '', false)).toBe(false); - }). - then(done); + await circleBuild(payload(BUILD)).then(h.verifyResponse(500)); + expect({ prNum: PR }).toExistAsAnArtifact(); + expect({ prNum: PR }).not.toExistAsABuild(); + expect({ prNum: PR, isPublic: false }).not.toExistAsABuild(); }); - it('should be able to notify that a PR has been updated (and do nothing)', done => { - prUpdated(+pr9). - then(h.verifyResponse(200)). - then(() => { - // The PR should still not exist. - expect(h.buildExists(pr9, '', false)).toBe(false); - expect(h.buildExists(pr9, '', true)).toBe(false); - }). - then(done); + it('should be able to notify that a PR has been updated (and do nothing)', async () => { + await prUpdated(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER).then(h.verifyResponse(200)); + // The PR should still not exist. + expect({ prNum: PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER, isPublic: false }).not.toExistAsABuild(); + expect({ prNum: PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER, isPublic: true }).not.toExistAsABuild(); }); }); @@ -103,215 +82,186 @@ h.runForAllSupportedSchemes((scheme, port) => describe(`integration (on ${scheme describe('for an existing PR', () => { - it('should be able to upload and serve a public build', done => { - const regexPrefix0 = `^PR: ${pr9} \\| SHA: ${sha0} \\| File:`; - const idxContentRegex0 = new RegExp(`${regexPrefix0} \\/index\\.html$`); - const barContentRegex0 = new RegExp(`${regexPrefix0} \\/foo\\/bar\\.js$`); + it('should be able to upload and serve a public build', async () => { + const BUILD = BuildNums.TRUST_CHECK_ACTIVE_TRUSTED_USER; + const PR = PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER; - const regexPrefix9 = `^PR: uploaded\\/${pr9} \\| SHA: ${sha9} \\| File:`; - const idxContentRegex9 = new RegExp(`${regexPrefix9} \\/index\\.html$`); - const barContentRegex9 = new RegExp(`${regexPrefix9} \\/foo\\/bar\\.js$`); + const regexPrefix1 = `^PR: ${PR} \\| SHA: ${ALT_SHA} \\| File:`; + const idxContentRegex1 = new RegExp(`${regexPrefix1} \\/index\\.html$`); + const barContentRegex1 = new RegExp(`${regexPrefix1} \\/foo\\/bar\\.js$`); - h.createDummyBuild(pr9, sha0); - h.createDummyArchive(pr9, sha9, archivePath); + const regexPrefix2 = `^BUILD: ${BUILD} \\| PR: ${PR} \\| SHA: ${SHA} \\| File:`; + const idxContentRegex2 = new RegExp(`${regexPrefix2} \\/index\\.html$`); + const barContentRegex2 = new RegExp(`${regexPrefix2} \\/foo\\/bar\\.js$`); - uploadBuild(pr9, sha9, archivePath). - then(() => Promise.all([ - getFile(pr9, sha0, 'index.html').then(h.verifyResponse(200, idxContentRegex0)), - getFile(pr9, sha0, 'foo/bar.js').then(h.verifyResponse(200, barContentRegex0)), - getFile(pr9, sha9, 'index.html').then(h.verifyResponse(200, idxContentRegex9)), - getFile(pr9, sha9, 'foo/bar.js').then(h.verifyResponse(200, barContentRegex9)), - ])). - then(done); + h.createDummyBuild(PR, ALT_SHA); + await circleBuild(payload(BUILD)).then(h.verifyResponse(201)); + await Promise.all([ + getFile(PR, ALT_SHA, 'index.html').then(h.verifyResponse(200, idxContentRegex1)), + getFile(PR, ALT_SHA, 'foo/bar.js').then(h.verifyResponse(200, barContentRegex1)), + getFile(PR, SHA, 'index.html').then(h.verifyResponse(200, idxContentRegex2)), + getFile(PR, SHA, 'foo/bar.js').then(h.verifyResponse(200, barContentRegex2)), + ]); + + expect({ prNum: PR, sha: SHA }).toExistAsABuild(); + expect({ prNum: PR, sha: ALT_SHA }).toExistAsABuild(); }); - it('should be able to upload but not serve a hidden build', done => { - const regexPrefix0 = `^PR: ${pr9} \\| SHA: ${sha0} \\| File:`; - const idxContentRegex0 = new RegExp(`${regexPrefix0} \\/index\\.html$`); - const barContentRegex0 = new RegExp(`${regexPrefix0} \\/foo\\/bar\\.js$`); + it('should be able to upload but not serve a hidden build', async () => { + const BUILD = BuildNums.TRUST_CHECK_UNTRUSTED; + const PR = PrNums.TRUST_CHECK_UNTRUSTED; - const regexPrefix9 = `^PR: uploaded\\/${pr9} \\| SHA: ${sha9} \\| File:`; - const idxContentRegex9 = new RegExp(`${regexPrefix9} \\/index\\.html$`); - const barContentRegex9 = new RegExp(`${regexPrefix9} \\/foo\\/bar\\.js$`); + h.createDummyBuild(PR, ALT_SHA, false); + await circleBuild(payload(BUILD)).then(h.verifyResponse(202)); - h.createDummyBuild(pr9, sha0, false); - h.createDummyArchive(pr9, sha9, archivePath); + await Promise.all([ + getFile(PR, ALT_SHA, 'index.html').then(h.verifyResponse(404)), + getFile(PR, ALT_SHA, 'foo/bar.js').then(h.verifyResponse(404)), + getFile(PR, SHA, 'index.html').then(h.verifyResponse(404)), + getFile(PR, SHA, 'foo/bar.js').then(h.verifyResponse(404)), + ]); - uploadBuild(pr9, sha9, archivePath, c.BV_verify_verifiedNotTrusted). - then(() => Promise.all([ - getFile(pr9, sha0, 'index.html').then(h.verifyResponse(404)), - getFile(pr9, sha0, 'foo/bar.js').then(h.verifyResponse(404)), - getFile(pr9, sha9, 'index.html').then(h.verifyResponse(404)), - getFile(pr9, sha9, 'foo/bar.js').then(h.verifyResponse(404)), - ])). - then(() => { - expect(h.buildExists(pr9, sha9)).toBe(false); - expect(h.buildExists(pr9, sha9, false)).toBe(true); - expect(h.readBuildFile(pr9, sha0, 'index.html', false)).toMatch(idxContentRegex0); - expect(h.readBuildFile(pr9, sha0, 'foo/bar.js', false)).toMatch(barContentRegex0); - expect(h.readBuildFile(pr9, sha9, 'index.html', false)).toMatch(idxContentRegex9); - expect(h.readBuildFile(pr9, sha9, 'foo/bar.js', false)).toMatch(barContentRegex9); - }). - then(done); + expect({ prNum: PR, sha: SHA }).not.toExistAsABuild(); + expect({ prNum: PR, sha: SHA, isPublic: false }).toExistAsABuild(); + expect({ prNum: PR, sha: ALT_SHA }).not.toExistAsABuild(); + expect({ prNum: PR, sha: ALT_SHA, isPublic: false }).toExistAsABuild(); }); - it('should reject an upload if verification fails', done => { - const errorRegex9 = new RegExp(`Error while verifying upload for PR ${pr9}: Test`); + it('should reject an upload if verification fails', async () => { + const BUILD = BuildNums.TRUST_CHECK_ERROR; + const PR = PrNums.TRUST_CHECK_ERROR; - h.createDummyBuild(pr9, sha0); - h.createDummyArchive(pr9, sha9, archivePath); + h.createDummyBuild(PR, ALT_SHA, false); - uploadBuild(pr9, sha9, archivePath, c.BV_verify_error). - then(h.verifyResponse(403, errorRegex9)). - then(() => { - expect(h.buildExists(pr9)).toBe(true); - expect(h.buildExists(pr9, sha0)).toBe(true); - expect(h.buildExists(pr9, sha9)).toBe(false); - }). - then(done); + await circleBuild(payload(BUILD)).then(h.verifyResponse(500)); + expect({ prNum: PR }).toExistAsAnArtifact(); + expect({ prNum: PR }).not.toExistAsABuild(); + expect({ prNum: PR, isPublic: false }).not.toExistAsABuild(); + expect({ prNum: PR, sha: ALT_SHA, isPublic: false }).toExistAsABuild(); }); - it('should not be able to overwrite an existing public build', done => { - const regexPrefix9 = `^PR: ${pr9} \\| SHA: ${sha9} \\| File:`; - const idxContentRegex9 = new RegExp(`${regexPrefix9} \\/index\\.html$`); - const barContentRegex9 = new RegExp(`${regexPrefix9} \\/foo\\/bar\\.js$`); + it('should not be able to overwrite an existing public build', async () => { + const BUILD = BuildNums.TRUST_CHECK_ACTIVE_TRUSTED_USER; + const PR = PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER; - h.createDummyBuild(pr9, sha9); - h.createDummyArchive(pr9, sha9, archivePath); + const regexPrefix = `^PR: ${PR} \\| SHA: ${SHA} \\| File:`; + const idxContentRegex = new RegExp(`${regexPrefix} \\/index\\.html$`); + const barContentRegex = new RegExp(`${regexPrefix} \\/foo\\/bar\\.js$`); - uploadBuild(pr9, sha9, archivePath). - then(h.verifyResponse(409)). - then(() => Promise.all([ - getFile(pr9, sha9, 'index.html').then(h.verifyResponse(200, idxContentRegex9)), - getFile(pr9, sha9, 'foo/bar.js').then(h.verifyResponse(200, barContentRegex9)), - ])). - then(done); + h.createDummyBuild(PR, SHA); + + await circleBuild(payload(BUILD)).then(h.verifyResponse(409)); + await Promise.all([ + getFile(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER, SHA, 'index.html').then(h.verifyResponse(200, idxContentRegex)), + getFile(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER, SHA, 'foo/bar.js').then(h.verifyResponse(200, barContentRegex)), + ]); + + expect({ prNum: PR }).toExistAsAnArtifact(); + expect({ prNum: PR }).toExistAsABuild(); }); - it('should not be able to overwrite an existing hidden build', done => { - const regexPrefix9 = `^PR: ${pr9} \\| SHA: ${sha9} \\| File:`; - const idxContentRegex9 = new RegExp(`${regexPrefix9} \\/index\\.html$`); - const barContentRegex9 = new RegExp(`${regexPrefix9} \\/foo\\/bar\\.js$`); + it('should not be able to overwrite an existing hidden build', async () => { + const BUILD = BuildNums.TRUST_CHECK_UNTRUSTED; + const PR = PrNums.TRUST_CHECK_UNTRUSTED; + h.createDummyBuild(PR, SHA, false); - h.createDummyBuild(pr9, sha9, false); - h.createDummyArchive(pr9, sha9, archivePath); + await circleBuild(payload(BUILD)).then(h.verifyResponse(409)); - uploadBuild(pr9, sha9, archivePath, c.BV_verify_verifiedNotTrusted). - then(h.verifyResponse(409)). - then(() => { - expect(h.readBuildFile(pr9, sha9, 'index.html', false)).toMatch(idxContentRegex9); - expect(h.readBuildFile(pr9, sha9, 'foo/bar.js', false)).toMatch(barContentRegex9); - }). - then(done); + expect({ prNum: PR }).toExistAsAnArtifact(); + expect({ prNum: PR, isPublic: false }).toExistAsABuild(); }); - it('should be able to request re-checking visibility (if outdated)', done => { - const publicPr = pr9; - const hiddenPr = String(c.BV_getPrIsTrusted_notTrusted); + it('should be able to request re-checking visibility (if outdated)', async () => { + const publicPr = PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER; + const hiddenPr = PrNums.TRUST_CHECK_UNTRUSTED; - h.createDummyBuild(publicPr, sha9, false); - h.createDummyBuild(hiddenPr, sha9, true); + h.createDummyBuild(publicPr, SHA, false); + h.createDummyBuild(hiddenPr, SHA, true); // PR visibilities are outdated (i.e. the opposte of what the should). - expect(h.buildExists(publicPr, '', false)).toBe(true); - expect(h.buildExists(publicPr, '', true)).toBe(false); - expect(h.buildExists(hiddenPr, '', false)).toBe(false); - expect(h.buildExists(hiddenPr, '', true)).toBe(true); + expect({ prNum: publicPr, sha: SHA, isPublic: false }).toExistAsABuild(false); + expect({ prNum: publicPr, sha: SHA, isPublic: true }).not.toExistAsABuild(false); + expect({ prNum: hiddenPr, sha: SHA, isPublic: false }).not.toExistAsABuild(false); + expect({ prNum: hiddenPr, sha: SHA, isPublic: true }).toExistAsABuild(false); - Promise. - all([ - prUpdated(+publicPr).then(h.verifyResponse(200)), - prUpdated(+hiddenPr).then(h.verifyResponse(200)), - ]). - then(() => { - // PR visibilities should have been updated. - expect(h.buildExists(publicPr, '', false)).toBe(false); - expect(h.buildExists(publicPr, '', true)).toBe(true); - expect(h.buildExists(hiddenPr, '', false)).toBe(true); - expect(h.buildExists(hiddenPr, '', true)).toBe(false); - }). - then(() => { - h.deletePrDir(publicPr, true); - h.deletePrDir(hiddenPr, false); - }). - then(done); + await Promise.all([ + prUpdated(publicPr).then(h.verifyResponse(200)), + prUpdated(hiddenPr).then(h.verifyResponse(200)), + ]); + + // PR visibilities should have been updated. + expect({ prNum: publicPr, isPublic: false }).not.toExistAsABuild(); + expect({ prNum: publicPr, isPublic: true }).toExistAsABuild(); + expect({ prNum: hiddenPr, isPublic: false }).toExistAsABuild(); + expect({ prNum: hiddenPr, isPublic: true }).not.toExistAsABuild(); }); - it('should be able to request re-checking visibility (if up-to-date)', done => { - const publicPr = pr9; - const hiddenPr = String(c.BV_getPrIsTrusted_notTrusted); + it('should be able to request re-checking visibility (if up-to-date)', async () => { + const publicPr = PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER; + const hiddenPr = PrNums.TRUST_CHECK_UNTRUSTED; - h.createDummyBuild(publicPr, sha9, true); - h.createDummyBuild(hiddenPr, sha9, false); + h.createDummyBuild(publicPr, SHA, true); + h.createDummyBuild(hiddenPr, SHA, false); // PR visibilities are already up-to-date. - expect(h.buildExists(publicPr, '', false)).toBe(false); - expect(h.buildExists(publicPr, '', true)).toBe(true); - expect(h.buildExists(hiddenPr, '', false)).toBe(true); - expect(h.buildExists(hiddenPr, '', true)).toBe(false); + expect({ prNum: publicPr, sha: SHA, isPublic: false }).not.toExistAsABuild(false); + expect({ prNum: publicPr, sha: SHA, isPublic: true }).toExistAsABuild(false); + expect({ prNum: hiddenPr, sha: SHA, isPublic: false }).toExistAsABuild(false); + expect({ prNum: hiddenPr, sha: SHA, isPublic: true }).not.toExistAsABuild(false); - Promise. - all([ - prUpdated(+publicPr).then(h.verifyResponse(200)), - prUpdated(+hiddenPr).then(h.verifyResponse(200)), - ]). - then(() => { - // PR visibilities are still up-to-date. - expect(h.buildExists(publicPr, '', false)).toBe(false); - expect(h.buildExists(publicPr, '', true)).toBe(true); - expect(h.buildExists(hiddenPr, '', false)).toBe(true); - expect(h.buildExists(hiddenPr, '', true)).toBe(false); - }). - then(done); + await Promise.all([ + prUpdated(publicPr).then(h.verifyResponse(200)), + prUpdated(hiddenPr).then(h.verifyResponse(200)), + ]); + + // PR visibilities are still up-to-date. + expect({ prNum: publicPr, isPublic: true }).toExistAsABuild(); + expect({ prNum: publicPr, isPublic: false }).not.toExistAsABuild(); + expect({ prNum: hiddenPr, isPublic: true }).not.toExistAsABuild(); + expect({ prNum: hiddenPr, isPublic: false }).toExistAsABuild(); }); - it('should reject a request if re-checking visibility fails', done => { - const errorPr = String(c.BV_getPrIsTrusted_error); + it('should reject a request if re-checking visibility fails', async () => { + const errorPr = PrNums.TRUST_CHECK_ERROR; - h.createDummyBuild(errorPr, sha9, true); + h.createDummyBuild(errorPr, SHA, true); - expect(h.buildExists(errorPr, '', false)).toBe(false); - expect(h.buildExists(errorPr, '', true)).toBe(true); + expect({ prNum: errorPr, isPublic: false }).not.toExistAsABuild(false); + expect({ prNum: errorPr, isPublic: true }).toExistAsABuild(false); - prUpdated(+errorPr). - then(h.verifyResponse(500, /Test/)). - then(() => { - // PR visibility should not have been updated. - expect(h.buildExists(errorPr, '', false)).toBe(false); - expect(h.buildExists(errorPr, '', true)).toBe(true); - }). - then(done); + await prUpdated(errorPr).then(h.verifyResponse(500, /TRUST_CHECK_ERROR/)); + + // PR visibility should not have been updated. + expect({ prNum: errorPr, isPublic: false }).not.toExistAsABuild(); + expect({ prNum: errorPr, isPublic: true }).toExistAsABuild(); }); - it('should reject a request if updating visibility fails', done => { + it('should reject a request if updating visibility fails', async () => { // One way to cause an error is to have both a public and a hidden directory for the same PR. - h.createDummyBuild(pr9, sha9, false); - h.createDummyBuild(pr9, sha9, true); + h.createDummyBuild(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER, SHA, false); + h.createDummyBuild(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER, SHA, true); - const hiddenPrDir = h.getPrDir(pr9, false); - const publicPrDir = h.getPrDir(pr9, true); + const hiddenPrDir = h.getPrDir(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER, false); + const publicPrDir = h.getPrDir(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER, true); const bodyRegex = new RegExp(`Request to move '${hiddenPrDir}' to existing directory '${publicPrDir}'`); - expect(h.buildExists(pr9, '', false)).toBe(true); - expect(h.buildExists(pr9, '', true)).toBe(true); + expect({ prNum: PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER, isPublic: false }).toExistAsABuild(false); + expect({ prNum: PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER, isPublic: true }).toExistAsABuild(false); - prUpdated(+pr9). - then(h.verifyResponse(409, bodyRegex)). - then(() => { - // PR visibility should not have been updated. - expect(h.buildExists(pr9, '', false)).toBe(true); - expect(h.buildExists(pr9, '', true)).toBe(true); - }). - then(done); + await prUpdated(PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER).then(h.verifyResponse(409, bodyRegex)); + + // PR visibility should not have been updated. + expect({ prNum: PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER, isPublic: false }).toExistAsABuild(); + expect({ prNum: PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER, isPublic: true }).toExistAsABuild(); }); }); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/start-test-upload-server.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/start-test-upload-server.ts index 3450d1bc0c..455e3ad4a2 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/start-test-upload-server.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/start-test-upload-server.ts @@ -1,38 +1,2 @@ -// Imports -import {GithubPullRequests} from '../common/github-pull-requests'; -import {BUILD_VERIFICATION_STATUS, BuildVerifier} from '../upload-server/build-verifier'; -import {UploadError} from '../upload-server/upload-error'; -import * as c from './constants'; - -// Run -// TODO(gkalpak): Add e2e tests to cover these interactions as well. -GithubPullRequests.prototype.addComment = () => Promise.resolve(); -BuildVerifier.prototype.getPrIsTrusted = (pr: number) => { - switch (pr) { - case c.BV_getPrIsTrusted_error: - // For e2e tests, fake an error. - return Promise.reject('Test'); - case c.BV_getPrIsTrusted_notTrusted: - // For e2e tests, fake an untrusted PR (`false`). - return Promise.resolve(false); - default: - // For e2e tests, default to trusted PRs (`true`). - return Promise.resolve(true); - } -}; -BuildVerifier.prototype.verify = (expectedPr: number, authHeader: string) => { - switch (authHeader) { - case c.BV_verify_error: - // For e2e tests, fake a verification error. - return Promise.reject(new UploadError(403, `Error while verifying upload for PR ${expectedPr}: Test`)); - case c.BV_verify_verifiedNotTrusted: - // For e2e tests, fake a `verifiedNotTrusted` verification status. - return Promise.resolve(BUILD_VERIFICATION_STATUS.verifiedNotTrusted); - default: - // For e2e tests, default to `verifiedAndTrusted` verification status. - return Promise.resolve(BUILD_VERIFICATION_STATUS.verifiedAndTrusted); - } -}; - -// tslint:disable-next-line: no-var-requires -require('../upload-server/index'); +import '../upload-server'; +import './mock-external-apis'; diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/tar-stream.d.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/tar-stream.d.ts new file mode 100644 index 0000000000..99a50ab35d --- /dev/null +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/tar-stream.d.ts @@ -0,0 +1,30 @@ +declare module 'tar-stream' { + + import {Readable, Writable} from 'stream'; + + export interface Pack extends Readable { + entry(header: Header, callback?: (err?: any) => {}): Writable; + entry(header: Header, contents: string, callback?: (err?: any) => {}): Writable; + entry(header: Header, buffer: Buffer, callback?: (err?: any) => {}): Writable; + entry(header: Header, buffer: string|Buffer, callback?: (err?: any) => {}): Writable; + finalize(); + destroy(err: any); + } + + export interface Header { + name: string; + mode?: number; + uid?: number; + gid?: number; + size?: number; + mtime?: Date; + type?: type; + linkname?: string; + uname?: string; + gname?: string; + devmajor?: number; + devminor?: number; + } + + export function pack(): Pack; +} diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/upload-server.e2e.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/upload-server.e2e.ts index 002d93de3e..2387a36a99 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/upload-server.e2e.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/lib/verify-setup/upload-server.e2e.ts @@ -1,235 +1,163 @@ // Imports import * as fs from 'fs'; -import * as path from 'path'; -import * as c from './constants'; -import {CmdResult, helper as h} from './helper'; +import {join} from 'path'; +import {AIO_UPLOAD_HOSTNAME, AIO_UPLOAD_PORT, AIO_WWW_USER} from '../common/env-variables'; +import {computeShortSha} from '../common/utils'; +import {ALT_SHA, BuildNums, PrNums, SHA, SIMILAR_SHA} from './constants'; +import {helper as h, makeCurl, payload} from './helper'; +import {customMatchers} from './jasmine-custom-matchers'; // Tests -describe('upload-server (on HTTP)', () => { - const hostname = h.uploadHostname; - const port = h.uploadPort; - const host = `${hostname}:${port}`; - const pr = '9'; - const sha9 = '9'.repeat(40); - const sha0 = '0'.repeat(40); +describe('upload-server', () => { + const hostname = AIO_UPLOAD_HOSTNAME; + const port = AIO_UPLOAD_PORT; + const host = `http://${hostname}:${port}`; - beforeEach(() => jasmine.DEFAULT_TIMEOUT_INTERVAL = 10000); + beforeEach(() => jasmine.DEFAULT_TIMEOUT_INTERVAL = 5000); + beforeEach(() => jasmine.addMatchers(customMatchers)); afterEach(() => h.cleanUp()); - describe(`${host}/create-build//`, () => { - const authorizationHeader = `--header "Authorization: Token FOO"`; - const xFileHeader = `--header "X-File: ${h.buildsDir}/snapshot.tar.gz"`; - const defaultHeaders = `${authorizationHeader} ${xFileHeader}`; - const curl = (url: string, headers = defaultHeaders) => `curl -iL ${headers} ${url}`; + describe(`${host}/circle-build`, () => { + const curl = makeCurl(`${host}/circle-build`); - it('should disallow non-GET requests', done => { - const url = `http://${host}/create-build/${pr}/${sha9}`; + it('should disallow non-POST requests', async () => { const bodyRegex = /^Unknown resource/; - Promise.all([ - h.runCmd(`curl -iLX PUT ${url}`).then(h.verifyResponse(404, bodyRegex)), - h.runCmd(`curl -iLX POST ${url}`).then(h.verifyResponse(404, bodyRegex)), - h.runCmd(`curl -iLX PATCH ${url}`).then(h.verifyResponse(404, bodyRegex)), - h.runCmd(`curl -iLX DELETE ${url}`).then(h.verifyResponse(404, bodyRegex)), - ]).then(done); + await Promise.all([ + curl({method: 'GET'}).then(h.verifyResponse(404, bodyRegex)), + curl({method: 'PUT'}).then(h.verifyResponse(404, bodyRegex)), + curl({method: 'PATCH'}).then(h.verifyResponse(404, bodyRegex)), + curl({method: 'DELETE'}).then(h.verifyResponse(404, bodyRegex)), + ]); }); - it('should reject requests without an \'AUTHORIZATION\' header', done => { - const headers1 = ''; - const headers2 = '--header "AUTHORIXATION: "'; - const url = `http://${host}/create-build/${pr}/${sha9}`; - const bodyRegex = /^Missing or empty 'AUTHORIZATION' header/; - - Promise.all([ - h.runCmd(curl(url, headers1)).then(h.verifyResponse(401, bodyRegex)), - h.runCmd(curl(url, headers2)).then(h.verifyResponse(401, bodyRegex)), - ]).then(done); + it('should respond with 404 for unknown paths', async () => { + await Promise.all([ + curl({url: `${host}/foo/circle-build`}).then(h.verifyResponse(404)), + curl({url: `${host}/foo-circle-build`}).then(h.verifyResponse(404)), + curl({url: `${host}/fooncircle-build`}).then(h.verifyResponse(404)), + curl({url: `${host}/circle-build/foo`}).then(h.verifyResponse(404)), + curl({url: `${host}/circle-build-foo`}).then(h.verifyResponse(404)), + curl({url: `${host}/circle-buildnfoo`}).then(h.verifyResponse(404)), + curl({url: `${host}/circle-build/pr`}).then(h.verifyResponse(404)), + curl({url: `${host}/circle-build42`}).then(h.verifyResponse(404)), + ]); }); - - it('should reject requests without an \'X-FILE\' header', done => { - const headers1 = authorizationHeader; - const headers2 = `${authorizationHeader} --header "X-FILE: "`; - const url = `http://${host}/create-build/${pr}/${sha9}`; - const bodyRegex = /^Missing or empty 'X-FILE' header/; - - Promise.all([ - h.runCmd(curl(url, headers1)).then(h.verifyResponse(400, bodyRegex)), - h.runCmd(curl(url, headers2)).then(h.verifyResponse(400, bodyRegex)), - ]).then(done); + it('should respond with 400 if the body is not valid', async () => { + await Promise.all([ + curl({ data: '' }).then(h.verifyResponse(400)), + curl({ data: {} }).then(h.verifyResponse(400)), + curl({ data: { payload: {} } }).then(h.verifyResponse(400)), + curl({ data: { payload: { build_num: 1 } } }).then(h.verifyResponse(400)), + curl({ data: { payload: { build_num: 1, build_parameters: {} } } }).then(h.verifyResponse(400)), + curl(payload(0)).then(h.verifyResponse(400)), + curl(payload(-1)).then(h.verifyResponse(400)), + ]); }); - - it('should reject requests for which the PR verification fails', done => { - const headers = `--header "Authorization: ${c.BV_verify_error}" ${xFileHeader}`; - const url = `http://${host}/create-build/${pr}/${sha9}`; - const bodyRegex = new RegExp(`Error while verifying upload for PR ${pr}: Test`); - - h.runCmd(curl(url, headers)). - then(h.verifyResponse(403, bodyRegex)). - then(done); + it('should respond with 500 if the CircleCI API request errors', async () => { + await curl(payload(BuildNums.BUILD_INFO_ERROR)).then(h.verifyResponse(500)); + await curl(payload(BuildNums.BUILD_INFO_404)).then(h.verifyResponse(500)); }); - - it('should respond with 404 for unknown paths', done => { - const cmdPrefix = curl(`http://${host}`); - - Promise.all([ - h.runCmd(`${cmdPrefix}/foo/create-build/${pr}/${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/foo-create-build/${pr}/${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/fooncreate-build/${pr}/${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/create-build/foo/${pr}/${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/create-build-foo/${pr}/${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/create-buildnfoo/${pr}/${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/create-build/pr${pr}/${sha9}`).then(h.verifyResponse(404)), - h.runCmd(`${cmdPrefix}/create-build/${pr}/${sha9}42`).then(h.verifyResponse(404)), - ]).then(done); + it('should respond with 204 if the build on CircleCI failed', async () => { + await curl(payload(BuildNums.BUILD_INFO_BUILD_FAILED)).then(h.verifyResponse(204)); }); - - it('should reject PRs with leading zeros', done => { - h.runCmd(curl(`http://${host}/create-build/0${pr}/${sha9}`)). - then(h.verifyResponse(404)). - then(done); + it('should respond with 500 if the github org from CircleCI does not match what is configured', async () => { + await curl(payload(BuildNums.BUILD_INFO_INVALID_GH_ORG)).then(h.verifyResponse(500)); }); - - it('should accept SHAs with leading zeros (but not trim the zeros)', done => { - Promise.all([ - h.runCmd(curl(`http://${host}/create-build/${pr}/0${sha9}`)).then(h.verifyResponse(404)), - h.runCmd(curl(`http://${host}/create-build/${pr}/${sha9}`)).then(h.verifyResponse(500)), - h.runCmd(curl(`http://${host}/create-build/${pr}/${sha0}`)).then(h.verifyResponse(500)), - ]).then(done); + it('should respond with 500 if the github repo from CircleCI does not match what is configured', async () => { + await curl(payload(BuildNums.BUILD_INFO_INVALID_GH_REPO)).then(h.verifyResponse(500)); }); + it('should respond with 500 if the github files API errors', async () => { + await curl(payload(BuildNums.CHANGED_FILES_ERROR)).then(h.verifyResponse(500)); + await curl(payload(BuildNums.CHANGED_FILES_404)).then(h.verifyResponse(500)); + }); - [true, false].forEach(isPublic => describe(`(for ${isPublic ? 'public' : 'hidden'} builds)`, () => { - const authorizationHeader2 = isPublic ? - authorizationHeader : `--header "Authorization: ${c.BV_verify_verifiedNotTrusted}"`; - const cmdPrefix = curl('', `${authorizationHeader2} ${xFileHeader}`); - const overwriteRe = RegExp(`^Request to overwrite existing ${isPublic ? 'public' : 'non-public'} directory`); + it('should respond with 204 if no significant files are changed by the PR', async () => { + await curl(payload(BuildNums.CHANGED_FILES_NONE)).then(h.verifyResponse(204)); + }); + it('should respond with 500 if the CircleCI artifact API fails', async () => { + await curl(payload(BuildNums.BUILD_ARTIFACTS_ERROR)).then(h.verifyResponse(500)); + await curl(payload(BuildNums.BUILD_ARTIFACTS_404)).then(h.verifyResponse(500)); + await curl(payload(BuildNums.BUILD_ARTIFACTS_EMPTY)).then(h.verifyResponse(500)); + await curl(payload(BuildNums.BUILD_ARTIFACTS_MISSING)).then(h.verifyResponse(500)); + }); - it('should not overwrite existing builds', done => { - h.createDummyBuild(pr, sha9, isPublic); - expect(h.readBuildFile(pr, sha9, 'index.html', isPublic)).toContain('index.html'); + it('should respond with 500 if fetching the artifact errors', async () => { + await curl(payload(BuildNums.DOWNLOAD_ARTIFACT_ERROR)).then(h.verifyResponse(500)); + await curl(payload(BuildNums.DOWNLOAD_ARTIFACT_404)).then(h.verifyResponse(500)); + }); - h.writeBuildFile(pr, sha9, 'index.html', 'My content', isPublic); - expect(h.readBuildFile(pr, sha9, 'index.html', isPublic)).toBe('My content'); + it('should respond with 500 if the GH trusted API fails', async () => { + await curl(payload(BuildNums.TRUST_CHECK_ERROR)).then(h.verifyResponse(500)); + expect({ prNum: PrNums.TRUST_CHECK_ERROR }).toExistAsAnArtifact(); + }); - h.runCmd(`${cmdPrefix} http://${host}/create-build/${pr}/${sha9}`). - then(h.verifyResponse(409, overwriteRe)). - then(() => expect(h.readBuildFile(pr, sha9, 'index.html', isPublic)).toBe('My content')). - then(done); - }); + it('should respond with 201 if a new public build is created', async () => { + await curl(payload(BuildNums.TRUST_CHECK_ACTIVE_TRUSTED_USER)) + .then(h.verifyResponse(201)); + expect({ prNum: PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER }).toExistAsABuild(); + }); + it('should respond with 202 if a new private build is created', async () => { + await curl(payload(BuildNums.TRUST_CHECK_UNTRUSTED)).then(h.verifyResponse(202)); + expect({ prNum: PrNums.TRUST_CHECK_UNTRUSTED, isPublic: false }).toExistAsABuild(); + }); - it('should not overwrite existing builds (even if the SHA is different)', done => { - // Since only the first few characters of the SHA are used, it is possible for two different - // SHAs to correspond to the same directory. In that case, we don't want the second SHA to - // overwrite the first. + [true].forEach(isPublic => { + const build = isPublic ? BuildNums.TRUST_CHECK_ACTIVE_TRUSTED_USER : BuildNums.TRUST_CHECK_UNTRUSTED; + const prNum = isPublic ? PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER : PrNums.TRUST_CHECK_UNTRUSTED; + const label = isPublic ? 'public' : 'non-public'; + const overwriteRe = RegExp(`^Request to overwrite existing ${label} directory`); + const statusCode = isPublic ? 201 : 202; - const sha9Almost = sha9.replace(/.$/, '8'); - expect(sha9Almost).not.toBe(sha9); + describe(`for ${label} builds`, () => { - h.createDummyBuild(pr, sha9, isPublic); - expect(h.readBuildFile(pr, sha9, 'index.html', isPublic)).toContain('index.html'); - - h.writeBuildFile(pr, sha9, 'index.html', 'My content', isPublic); - expect(h.readBuildFile(pr, sha9, 'index.html', isPublic)).toBe('My content'); - - h.runCmd(`${cmdPrefix} http://${host}/create-build/${pr}/${sha9Almost}`). - then(h.verifyResponse(409, overwriteRe)). - then(() => expect(h.readBuildFile(pr, sha9, 'index.html', isPublic)).toBe('My content')). - then(done); - }); - - - it('should delete the PR directory on error (for new PR)', done => { - h.runCmd(`${cmdPrefix} http://${host}/create-build/${pr}/${sha9}`). - then(h.verifyResponse(500)). - then(() => expect(h.buildExists(pr, '', isPublic)).toBe(false)). - then(done); - }); - - - it('should only delete the SHA directory on error (for existing PR)', done => { - h.createDummyBuild(pr, sha0, isPublic); - - h.runCmd(`${cmdPrefix} http://${host}/create-build/${pr}/${sha9}`). - then(h.verifyResponse(500)). - then(() => { - expect(h.buildExists(pr, sha9, isPublic)).toBe(false); - expect(h.buildExists(pr, '', isPublic)).toBe(true); - }). - then(done); - }); - - - describe('on successful upload', () => { - const archivePath = path.join(h.buildsDir, 'snapshot.tar.gz'); - const statusCode = isPublic ? 201 : 202; - let uploadPromise: Promise; - - beforeEach(() => { - h.createDummyArchive(pr, sha9, archivePath); - uploadPromise = h.runCmd(`${cmdPrefix} http://${host}/create-build/${pr}/${sha9}`); - }); - afterEach(() => h.deletePrDir(pr, isPublic)); - - - it(`should respond with ${statusCode}`, done => { - uploadPromise.then(h.verifyResponse(statusCode)).then(done); + it('should extract the contents of the uploaded file', async () => { + await curl(payload(build)) + .then(h.verifyResponse(statusCode)); + expect(h.readBuildFile(prNum, SHA, 'index.html', isPublic)) + .toContain(`PR: ${prNum} | SHA: ${SHA} | File: /index.html`); + expect(h.readBuildFile(prNum, SHA, 'foo/bar.js', isPublic)) + .toContain(`PR: ${prNum} | SHA: ${SHA} | File: /foo/bar.js`); + expect({ prNum, isPublic }).toExistAsABuild(); }); + it(`should create files/directories owned by '${AIO_WWW_USER}'`, async () => { + await curl(payload(build)) + .then(h.verifyResponse(statusCode)); - it('should extract the contents of the uploaded file', done => { - uploadPromise. - then(() => { - expect(h.readBuildFile(pr, sha9, 'index.html', isPublic)).toContain(`uploaded/${pr}`); - expect(h.readBuildFile(pr, sha9, 'foo/bar.js', isPublic)).toContain(`uploaded/${pr}`); - }). - then(done); + const shaDir = h.getShaDir(h.getPrDir(prNum, isPublic), SHA); + const { stdout: allFiles } = await h.runCmd(`find ${shaDir}`); + const { stdout: userFiles } = await h.runCmd(`find ${shaDir} -user ${AIO_WWW_USER}`); + + expect(userFiles).toBe(allFiles); + expect(userFiles).toContain(shaDir); + expect(userFiles).toContain(join(shaDir, 'index.html')); + expect(userFiles).toContain(join(shaDir, 'foo', 'bar.js')); + + expect({ prNum, isPublic }).toExistAsABuild(); }); - - it(`should create files/directories owned by '${h.wwwUser}'`, done => { - const prDir = h.getPrDir(pr, isPublic); - const shaDir = h.getShaDir(prDir, sha9); - const idxPath = path.join(shaDir, 'index.html'); - const barPath = path.join(shaDir, 'foo', 'bar.js'); - - uploadPromise. - then(() => Promise.all([ - h.runCmd(`find ${shaDir}`), - h.runCmd(`find ${shaDir} -user ${h.wwwUser}`), - ])). - then(([{stdout: allFiles}, {stdout: userFiles}]) => { - expect(userFiles).toBe(allFiles); - expect(userFiles).toContain(shaDir); - expect(userFiles).toContain(idxPath); - expect(userFiles).toContain(barPath); - }). - then(done); + it('should delete the uploaded file', async () => { + await curl(payload(build)) + .then(h.verifyResponse(statusCode)); + expect({ prNum, SHA }).not.toExistAsAnArtifact(); + expect({ prNum, isPublic }).toExistAsABuild(); }); - - it('should delete the uploaded file', done => { - expect(fs.existsSync(archivePath)).toBe(true); - uploadPromise. - then(() => expect(fs.existsSync(archivePath)).toBe(false)). - then(done); - }); - - - it('should make the build directory non-writable', done => { - const prDir = h.getPrDir(pr, isPublic); - const shaDir = h.getShaDir(prDir, sha9); - const idxPath = path.join(shaDir, 'index.html'); - const barPath = path.join(shaDir, 'foo', 'bar.js'); + it('should make the build directory non-writable', async () => { + await curl(payload(build)) + .then(h.verifyResponse(statusCode)); // See https://github.com/nodejs/node-v0.x-archive/issues/3045#issuecomment-4862588. const isNotWritable = (fileOrDir: string) => { @@ -238,116 +166,113 @@ describe('upload-server (on HTTP)', () => { return !(mode & parseInt('222', 8)); }; - uploadPromise. - then(() => { - expect(isNotWritable(shaDir)).toBe(true); - expect(isNotWritable(idxPath)).toBe(true); - expect(isNotWritable(barPath)).toBe(true); - }). - then(done); + const shaDir = h.getShaDir(h.getPrDir(prNum, isPublic), SHA); + expect(isNotWritable(shaDir)).toBe(true); + expect(isNotWritable(join(shaDir, 'index.html'))).toBe(true); + expect(isNotWritable(join(shaDir, 'foo', 'bar.js'))).toBe(true); + + expect({ prNum, isPublic }).toExistAsABuild(); }); - - it('should ignore a legacy 40-chars long build directory (even if it starts with the same chars)', done => { + it('should ignore a legacy 40-chars long build directory (even if it starts with the same chars)', + async () => { // It is possible that 40-chars long build directories exist, if they had been deployed // before implementing the shorter build directory names. In that case, we don't want the // second (shorter) name to be considered the same as the old one (even if they originate // from the same SHA). - h.createDummyBuild(pr, sha9, isPublic, false, true); - expect(h.readBuildFile(pr, sha9, 'index.html', isPublic, true)).toContain('index.html'); + h.createDummyBuild(prNum, SHA, isPublic, false, true); + h.writeBuildFile(prNum, SHA, 'index.html', 'My content', isPublic, true); + expect(h.readBuildFile(prNum, SHA, 'index.html', isPublic, true)).toBe('My content'); - h.writeBuildFile(pr, sha9, 'index.html', 'My content', isPublic, true); - expect(h.readBuildFile(pr, sha9, 'index.html', isPublic, true)).toBe('My content'); + await curl(payload(build)) + .then(h.verifyResponse(statusCode)); - h.runCmd(`${cmdPrefix} http://${host}/create-build/${pr}/${sha9}`). - then(h.verifyResponse(statusCode)). - then(() => { - expect(h.buildExists(pr, sha9, isPublic)).toBe(true); - expect(h.buildExists(pr, sha9, isPublic, true)).toBe(true); - expect(h.readBuildFile(pr, sha9, 'index.html', isPublic)).toContain('index.html'); - expect(h.readBuildFile(pr, sha9, 'index.html', isPublic, true)).toBe('My content'); - }). - then(done); + expect(h.readBuildFile(prNum, SHA, 'index.html', isPublic, false)).toContain('index.html'); + expect(h.readBuildFile(prNum, SHA, 'index.html', isPublic, true)).toBe('My content'); + + expect({ prNum, isPublic, sha: SHA, isLegacy: false }).toExistAsABuild(); + expect({ prNum, isPublic, sha: SHA, isLegacy: true }).toExistAsABuild(); }); + it(`should not overwrite existing builds`, async () => { + // setup a build already in place + h.createDummyBuild(prNum, SHA, isPublic); + // distinguish this build from the downloaded one + h.writeBuildFile(prNum, SHA, 'index.html', 'My content', isPublic); + await curl(payload(build)).then(h.verifyResponse(409, overwriteRe)); + expect(h.readBuildFile(prNum, SHA, 'index.html', isPublic)).toBe('My content'); + expect({ prNum, isPublic }).toExistAsABuild(); + expect({ prNum }).toExistAsAnArtifact(); + }); + + it(`should not overwrite existing builds (even if the SHA is different)`, async () => { + // Since only the first few characters of the SHA are used, it is possible for two different + // SHAs to correspond to the same directory. In that case, we don't want the second SHA to + // overwrite the first. + expect(SIMILAR_SHA).not.toEqual(SHA); + expect(computeShortSha(SIMILAR_SHA)).toEqual(computeShortSha(SHA)); + h.createDummyBuild(prNum, SIMILAR_SHA, isPublic); + expect(h.readBuildFile(prNum, SIMILAR_SHA, 'index.html', isPublic)).toContain('index.html'); + h.writeBuildFile(prNum, SIMILAR_SHA, 'index.html', 'My content', isPublic); + expect(h.readBuildFile(prNum, SIMILAR_SHA, 'index.html', isPublic)).toBe('My content'); + + await curl(payload(build)).then(h.verifyResponse(409, overwriteRe)); + expect(h.readBuildFile(prNum, SIMILAR_SHA, 'index.html', isPublic)).toBe('My content'); + expect({ prNum, isPublic, sha: SIMILAR_SHA }).toExistAsABuild(); + expect({ prNum, sha: SIMILAR_SHA }).toExistAsAnArtifact(); + }); + + it('should only delete the SHA directory on error (for existing PR)', async () => { + h.createDummyBuild(prNum, ALT_SHA, isPublic); + await curl(payload(BuildNums.TRUST_CHECK_ERROR)).then(h.verifyResponse(500)); + expect({ prNum: PrNums.TRUST_CHECK_ERROR }).toExistAsAnArtifact(); + expect({ prNum, isPublic, sha: SHA }).not.toExistAsABuild(); + expect({ prNum, isPublic, sha: ALT_SHA }).toExistAsABuild(); + }); + + describe('when the PR\'s visibility has changed', () => { + + it('should update the PR\'s visibility', async () => { + h.createDummyBuild(prNum, ALT_SHA, !isPublic); + await curl(payload(build)).then(h.verifyResponse(statusCode)); + expect({ prNum, isPublic }).toExistAsABuild(); + expect({ prNum, isPublic, sha: ALT_SHA }).toExistAsABuild(); + }); + + + it('should not overwrite existing builds (but keep the updated visibility)', async () => { + h.createDummyBuild(prNum, SHA, !isPublic); + await curl(payload(build)).then(h.verifyResponse(409)); + expect({ prNum, isPublic }).toExistAsABuild(); + expect({ prNum, isPublic: !isPublic }).not.toExistAsABuild(); + // since it errored we didn't clear up the downloaded artifact - perhaps we should? + expect({ prNum }).toExistAsAnArtifact(); + }); + + + it('should reject the request if it fails to update the PR\'s visibility', async () => { + // One way to cause an error is to have both a public and a hidden directory for the same PR. + h.createDummyBuild(prNum, ALT_SHA, isPublic); + h.createDummyBuild(prNum, ALT_SHA, !isPublic); + + const errorRegex = new RegExp(`^Request to move '${h.getPrDir(prNum, !isPublic)}' ` + + `to existing directory '${h.getPrDir(prNum, isPublic)}'.`); + + await curl(payload(build)).then(h.verifyResponse(409, errorRegex)); + + expect({ prNum, isPublic }).not.toExistAsABuild(); + + // The bad folders should have been deleted + expect({ prNum, sha: ALT_SHA, isPublic }).toExistAsABuild(); + expect({ prNum, sha: ALT_SHA, isPublic: !isPublic }).toExistAsABuild(); + + // since it errored we didn't clear up the downloaded artifact - perhaps we should? + expect({ prNum }).toExistAsAnArtifact(); + }); + }); }); - - - describe('when the PR\'s visibility has changed', () => { - const archivePath = path.join(h.buildsDir, 'snapshot.tar.gz'); - const statusCode = isPublic ? 201 : 202; - - const checkPrVisibility = (isPublic2: boolean) => { - expect(h.buildExists(pr, '', isPublic2)).toBe(true); - expect(h.buildExists(pr, '', !isPublic2)).toBe(false); - expect(h.buildExists(pr, sha0, isPublic2)).toBe(true); - expect(h.buildExists(pr, sha0, !isPublic2)).toBe(false); - }; - const uploadBuild = (sha: string) => h.runCmd(`${cmdPrefix} http://${host}/create-build/${pr}/${sha}`); - - beforeEach(() => { - h.createDummyBuild(pr, sha0, !isPublic); - h.createDummyArchive(pr, sha9, archivePath); - checkPrVisibility(!isPublic); - }); - afterEach(() => h.deletePrDir(pr, isPublic)); - - - it('should update the PR\'s visibility', done => { - uploadBuild(sha9). - then(h.verifyResponse(statusCode)). - then(() => { - checkPrVisibility(isPublic); - expect(h.buildExists(pr, sha9, isPublic)).toBe(true); - expect(h.readBuildFile(pr, sha9, 'index.html', isPublic)).toContain(`uploaded/${pr}`); - expect(h.readBuildFile(pr, sha9, 'index.html', isPublic)).toContain(sha9); - }). - then(done); - }); - - - it('should not overwrite existing builds (but keep the updated visibility)', done => { - expect(h.buildExists(pr, sha0, isPublic)).toBe(false); - - uploadBuild(sha0). - then(h.verifyResponse(409, overwriteRe)). - then(() => { - checkPrVisibility(isPublic); - expect(h.readBuildFile(pr, sha0, 'index.html', isPublic)).toContain(pr); - expect(h.readBuildFile(pr, sha0, 'index.html', isPublic)).not.toContain(`uploaded/${pr}`); - expect(h.readBuildFile(pr, sha0, 'index.html', isPublic)).toContain(sha0); - expect(h.readBuildFile(pr, sha0, 'index.html', isPublic)).not.toContain(sha9); - }). - then(done); - }); - - - it('should reject the request if it fails to update the PR\'s visibility', done => { - // One way to cause an error is to have both a public and a hidden directory for the same PR. - h.createDummyBuild(pr, sha0, isPublic); - - expect(h.buildExists(pr, sha0, isPublic)).toBe(true); - expect(h.buildExists(pr, sha0, !isPublic)).toBe(true); - - const errorRegex = new RegExp(`^Request to move '${h.getPrDir(pr, !isPublic)}' ` + - `to existing directory '${h.getPrDir(pr, isPublic)}'.`); - - uploadBuild(sha9). - then(h.verifyResponse(409, errorRegex)). - then(() => { - expect(h.buildExists(pr, sha0, isPublic)).toBe(true); - expect(h.buildExists(pr, sha0, !isPublic)).toBe(true); - expect(h.buildExists(pr, sha9, isPublic)).toBe(false); - expect(h.buildExists(pr, sha9, !isPublic)).toBe(false); - }). - then(done); - }); - - }); - - })); - + }); }); @@ -355,20 +280,20 @@ describe('upload-server (on HTTP)', () => { it('should respond with 200', done => { Promise.all([ - h.runCmd(`curl -iL http://${host}/health-check`).then(h.verifyResponse(200)), - h.runCmd(`curl -iL http://${host}/health-check/`).then(h.verifyResponse(200)), + h.runCmd(`curl -iL ${host}/health-check`).then(h.verifyResponse(200)), + h.runCmd(`curl -iL ${host}/health-check/`).then(h.verifyResponse(200)), ]).then(done); }); it('should respond with 404 if the path does not match exactly', done => { Promise.all([ - h.runCmd(`curl -iL http://${host}/health-check/foo`).then(h.verifyResponse(404)), - h.runCmd(`curl -iL http://${host}/health-check-foo`).then(h.verifyResponse(404)), - h.runCmd(`curl -iL http://${host}/health-checknfoo`).then(h.verifyResponse(404)), - h.runCmd(`curl -iL http://${host}/foo/health-check`).then(h.verifyResponse(404)), - h.runCmd(`curl -iL http://${host}/foo-health-check`).then(h.verifyResponse(404)), - h.runCmd(`curl -iL http://${host}/foonhealth-check`).then(h.verifyResponse(404)), + h.runCmd(`curl -iL ${host}/health-check/foo`).then(h.verifyResponse(404)), + h.runCmd(`curl -iL ${host}/health-check-foo`).then(h.verifyResponse(404)), + h.runCmd(`curl -iL ${host}/health-checknfoo`).then(h.verifyResponse(404)), + h.runCmd(`curl -iL ${host}/foo/health-check`).then(h.verifyResponse(404)), + h.runCmd(`curl -iL ${host}/foo-health-check`).then(h.verifyResponse(404)), + h.runCmd(`curl -iL ${host}/foonhealth-check`).then(h.verifyResponse(404)), ]).then(done); }); @@ -376,56 +301,48 @@ describe('upload-server (on HTTP)', () => { describe(`${host}/pr-updated`, () => { - const url = `http://${host}/pr-updated`; + const curl = makeCurl(`${host}/pr-updated`); - // Helpers - const curl = (payload?: {number: number, action?: string}) => { - const payloadStr = payload && JSON.stringify(payload) || ''; - return `curl -iLX POST --header "Content-Type: application/json" --data '${payloadStr}' ${url}`; - }; - - - it('should disallow non-POST requests', done => { + it('should disallow non-POST requests', async () => { const bodyRegex = /^Unknown resource in request/; - Promise.all([ - h.runCmd(`curl -iLX GET ${url}`).then(h.verifyResponse(404, bodyRegex)), - h.runCmd(`curl -iLX PUT ${url}`).then(h.verifyResponse(404, bodyRegex)), - h.runCmd(`curl -iLX PATCH ${url}`).then(h.verifyResponse(404, bodyRegex)), - h.runCmd(`curl -iLX DELETE ${url}`).then(h.verifyResponse(404, bodyRegex)), - ]).then(done); + await Promise.all([ + curl({method: 'GET'}).then(h.verifyResponse(404, bodyRegex)), + curl({method: 'PUT'}).then(h.verifyResponse(404, bodyRegex)), + curl({method: 'PATCH'}).then(h.verifyResponse(404, bodyRegex)), + curl({method: 'DELETE'}).then(h.verifyResponse(404, bodyRegex)), + ]); }); - it('should respond with 400 for requests without a payload', done => { + it('should respond with 400 for requests without a payload', async () => { const bodyRegex = /^Missing or empty 'number' field in request/; - h.runCmd(curl()). - then(h.verifyResponse(400, bodyRegex)). - then(done); + await Promise.all([ + curl({ data: '' }).then(h.verifyResponse(400, bodyRegex)), + curl({ data: {} }).then(h.verifyResponse(400, bodyRegex)), + ]); }); - it('should respond with 400 for requests without a \'number\' field', done => { + it('should respond with 400 for requests without a \'number\' field', async () => { const bodyRegex = /^Missing or empty 'number' field in request/; - Promise.all([ - h.runCmd(curl({} as any)).then(h.verifyResponse(400, bodyRegex)), - h.runCmd(curl({number: null} as any)).then(h.verifyResponse(400, bodyRegex)), - ]).then(done); + await Promise.all([ + curl({ data: {} }).then(h.verifyResponse(400, bodyRegex)), + curl({ data: { number: null} }).then(h.verifyResponse(400, bodyRegex)), + ]); }); - it('should reject requests for which checking the PR visibility fails', done => { - h.runCmd(curl({number: c.BV_getPrIsTrusted_error})). - then(h.verifyResponse(500, /Test/)). - then(done); + it('should reject requests for which checking the PR visibility fails', async () => { + await curl({ data: { number: PrNums.TRUST_CHECK_ERROR } }).then(h.verifyResponse(500, /TRUST_CHECK_ERROR/)); }); it('should respond with 404 for unknown paths', done => { - const mockPayload = JSON.stringify({number: +pr}); - const cmdPrefix = `curl -iLX POST --data "${mockPayload}" http://${host}`; + const mockPayload = JSON.stringify({number: 1}); // MockExternalApiFlags.TRUST_CHECK_ACTIVE_TRUSTED_USER }); + const cmdPrefix = `curl -iLX POST --data "${mockPayload}" ${host}`; Promise.all([ h.runCmd(`${cmdPrefix}/foo/pr-updated`).then(h.verifyResponse(404)), @@ -438,111 +355,107 @@ describe('upload-server (on HTTP)', () => { }); - it('should do nothing if PR\'s visibility is already up-to-date', done => { - const publicPr = pr; - const hiddenPr = String(c.BV_getPrIsTrusted_notTrusted); - const checkVisibilities = () => { + it('should do nothing if PR\'s visibility is already up-to-date', async () => { + const publicPr = PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER; + const hiddenPr = PrNums.TRUST_CHECK_UNTRUSTED; + + const checkVisibilities = (remove: boolean) => { // Public build is already public. - expect(h.buildExists(publicPr, '', false)).toBe(false); - expect(h.buildExists(publicPr, '', true)).toBe(true); + expect({ prNum: publicPr, isPublic: false }).not.toExistAsABuild(remove); + expect({ prNum: publicPr, isPublic: true }).toExistAsABuild(remove); // Hidden build is already hidden. - expect(h.buildExists(hiddenPr, '', false)).toBe(true); - expect(h.buildExists(hiddenPr, '', true)).toBe(false); + expect({ prNum: hiddenPr, isPublic: false }).toExistAsABuild(remove); + expect({ prNum: hiddenPr, isPublic: true }).not.toExistAsABuild(remove); }; - h.createDummyBuild(publicPr, sha9, true); - h.createDummyBuild(hiddenPr, sha9, false); - checkVisibilities(); + h.createDummyBuild(publicPr, SHA, true); + h.createDummyBuild(hiddenPr, SHA, false); + checkVisibilities(false); - Promise. - all([ - h.runCmd(curl({number: +publicPr, action: 'foo'})).then(h.verifyResponse(200)), - h.runCmd(curl({number: +hiddenPr, action: 'foo'})).then(h.verifyResponse(200)), - ]). - // Visibilities should not have changed, because the specified action could not have triggered a change. - then(checkVisibilities). - then(done); + await Promise.all([ + curl({ data: {number: +publicPr, action: 'foo' } }).then(h.verifyResponse(200)), + curl({ data: {number: +hiddenPr, action: 'foo' } }).then(h.verifyResponse(200)), + ]); + + // Visibilities should not have changed, because the specified action could not have triggered a change. + checkVisibilities(true); }); - it('should do nothing if \'action\' implies no visibility change', done => { - const publicPr = pr; - const hiddenPr = String(c.BV_getPrIsTrusted_notTrusted); - const checkVisibilities = () => { + it('should do nothing if \'action\' implies no visibility change', async () => { + const publicPr = PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER; + const hiddenPr = PrNums.TRUST_CHECK_UNTRUSTED; + + const checkVisibilities = (remove: boolean) => { // Public build is hidden atm. - expect(h.buildExists(publicPr, '', false)).toBe(true); - expect(h.buildExists(publicPr, '', true)).toBe(false); + expect({ prNum: publicPr, isPublic: false }).toExistAsABuild(remove); + expect({ prNum: publicPr, isPublic: true }).not.toExistAsABuild(remove); // Hidden build is public atm. - expect(h.buildExists(hiddenPr, '', false)).toBe(false); - expect(h.buildExists(hiddenPr, '', true)).toBe(true); + expect({ prNum: hiddenPr, isPublic: false }).not.toExistAsABuild(remove); + expect({ prNum: hiddenPr, isPublic: true }).toExistAsABuild(remove); }; - h.createDummyBuild(publicPr, sha9, false); - h.createDummyBuild(hiddenPr, sha9, true); - checkVisibilities(); + h.createDummyBuild(publicPr, SHA, false); + h.createDummyBuild(hiddenPr, SHA, true); + checkVisibilities(false); - Promise. - all([ - h.runCmd(curl({number: +publicPr, action: 'foo'})).then(h.verifyResponse(200)), - h.runCmd(curl({number: +hiddenPr, action: 'foo'})).then(h.verifyResponse(200)), - ]). - // Visibilities should not have changed, because the specified action could not have triggered a change. - then(checkVisibilities). - then(done); + await Promise.all([ + curl({ data: {number: +publicPr, action: 'foo' } }).then(h.verifyResponse(200)), + curl({ data: {number: +hiddenPr, action: 'foo' } }).then(h.verifyResponse(200)), + ]); + // Visibilities should not have changed, because the specified action could not have triggered a change. + checkVisibilities(true); }); describe('when the visiblity has changed', () => { - const publicPr = pr; - const hiddenPr = String(c.BV_getPrIsTrusted_notTrusted); + const publicPr = PrNums.TRUST_CHECK_ACTIVE_TRUSTED_USER; + const hiddenPr = PrNums.TRUST_CHECK_UNTRUSTED; beforeEach(() => { // Create initial PR builds with opposite visibilities as the ones that will be reported: // - The now public PR was previously hidden. // - The now hidden PR was previously public. - h.createDummyBuild(publicPr, sha9, false); - h.createDummyBuild(hiddenPr, sha9, true); + h.createDummyBuild(publicPr, SHA, false); + h.createDummyBuild(hiddenPr, SHA, true); - expect(h.buildExists(publicPr, '', false)).toBe(true); - expect(h.buildExists(publicPr, '', true)).toBe(false); - expect(h.buildExists(hiddenPr, '', false)).toBe(false); - expect(h.buildExists(hiddenPr, '', true)).toBe(true); + expect({ prNum: publicPr, isPublic: false }).toExistAsABuild(false); + expect({ prNum: publicPr, isPublic: true }).not.toExistAsABuild(false); + expect({ prNum: hiddenPr, isPublic: false }).not.toExistAsABuild(false); + expect({ prNum: hiddenPr, isPublic: true }).toExistAsABuild(false); }); afterEach(() => { // Expect PRs' visibility to have been updated: // - The public PR should be actually public (previously it was hidden). // - The hidden PR should be actually hidden (previously it was public). - expect(h.buildExists(publicPr, '', false)).toBe(false); - expect(h.buildExists(publicPr, '', true)).toBe(true); - expect(h.buildExists(hiddenPr, '', false)).toBe(true); - expect(h.buildExists(hiddenPr, '', true)).toBe(false); - - h.deletePrDir(publicPr, true); - h.deletePrDir(hiddenPr, false); + expect({ prNum: publicPr, isPublic: false }).not.toExistAsABuild(); + expect({ prNum: publicPr, isPublic: true }).toExistAsABuild(); + expect({ prNum: hiddenPr, isPublic: false }).toExistAsABuild(); + expect({ prNum: hiddenPr, isPublic: true }).not.toExistAsABuild(); }); - it('should update the PR\'s visibility (action: undefined)', done => { - Promise.all([ - h.runCmd(curl({number: +publicPr})).then(h.verifyResponse(200)), - h.runCmd(curl({number: +hiddenPr})).then(h.verifyResponse(200)), - ]).then(done); + it('should update the PR\'s visibility (action: undefined)', async () => { + await Promise.all([ + curl({ data: {number: +publicPr } }).then(h.verifyResponse(200)), + curl({ data: {number: +hiddenPr } }).then(h.verifyResponse(200)), + ]); }); - it('should update the PR\'s visibility (action: labeled)', done => { - Promise.all([ - h.runCmd(curl({number: +publicPr, action: 'labeled'})).then(h.verifyResponse(200)), - h.runCmd(curl({number: +hiddenPr, action: 'labeled'})).then(h.verifyResponse(200)), - ]).then(done); + it('should update the PR\'s visibility (action: labeled)', async () => { + await Promise.all([ + curl({ data: {number: +publicPr, action: 'labeled' } }).then(h.verifyResponse(200)), + curl({ data: {number: +hiddenPr, action: 'labeled' } }).then(h.verifyResponse(200)), + ]); }); - it('should update the PR\'s visibility (action: unlabeled)', done => { - Promise.all([ - h.runCmd(curl({number: +publicPr, action: 'unlabeled'})).then(h.verifyResponse(200)), - h.runCmd(curl({number: +hiddenPr, action: 'unlabeled'})).then(h.verifyResponse(200)), - ]).then(done); + it('should update the PR\'s visibility (action: unlabeled)', async () => { + await Promise.all([ + curl({ data: {number: +publicPr, action: 'unlabeled' } }).then(h.verifyResponse(200)), + curl({ data: {number: +hiddenPr, action: 'unlabeled' } }).then(h.verifyResponse(200)), + ]); }); }); @@ -556,16 +469,15 @@ describe('upload-server (on HTTP)', () => { const bodyRegex = /^Unknown resource/; Promise.all([ - h.runCmd(`curl -iL http://${host}/index.html`).then(h.verifyResponse(404, bodyRegex)), - h.runCmd(`curl -iL http://${host}/`).then(h.verifyResponse(404, bodyRegex)), - h.runCmd(`curl -iL http://${host}`).then(h.verifyResponse(404, bodyRegex)), - h.runCmd(`curl -iLX PUT http://${host}`).then(h.verifyResponse(404, bodyRegex)), - h.runCmd(`curl -iLX POST http://${host}`).then(h.verifyResponse(404, bodyRegex)), - h.runCmd(`curl -iLX PATCH http://${host}`).then(h.verifyResponse(404, bodyRegex)), - h.runCmd(`curl -iLX DELETE http://${host}`).then(h.verifyResponse(404, bodyRegex)), + h.runCmd(`curl -iL ${host}/index.html`).then(h.verifyResponse(404, bodyRegex)), + h.runCmd(`curl -iL ${host}/`).then(h.verifyResponse(404, bodyRegex)), + h.runCmd(`curl -iL ${host}`).then(h.verifyResponse(404, bodyRegex)), + h.runCmd(`curl -iLX PUT ${host}`).then(h.verifyResponse(404, bodyRegex)), + h.runCmd(`curl -iLX POST ${host}`).then(h.verifyResponse(404, bodyRegex)), + h.runCmd(`curl -iLX PATCH ${host}`).then(h.verifyResponse(404, bodyRegex)), + h.runCmd(`curl -iLX DELETE ${host}`).then(h.verifyResponse(404, bodyRegex)), ]).then(done); }); }); - }); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/package.json b/aio/aio-builds-setup/dockerbuild/scripts-js/package.json index 05ba8a807c..6cc6e30f77 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/package.json +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/package.json @@ -21,18 +21,22 @@ }, "dependencies": { "body-parser": "^1.18.2", + "delete-empty": "^2.0.0", "express": "^4.15.4", "jasmine": "^2.8.0", - "jsonwebtoken": "^8.0.1", - "shelljs": "^0.7.8", + "nock": "^9.2.5", + "node-fetch": "^2.1.2", + "shelljs": "^0.8.1", + "tar-stream": "^1.6.0", "tslib": "^1.7.1" }, "devDependencies": { "@types/body-parser": "^1.16.5", "@types/express": "^4.0.37", "@types/jasmine": "^2.6.0", - "@types/jsonwebtoken": "^7.2.3", + "@types/nock": "^9.1.3", "@types/node": "^8.0.30", + "@types/node-fetch": "^1.6.8", "@types/shelljs": "^0.8.0", "@types/supertest": "^2.0.3", "concurrently": "^3.5.0", diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/test/clean-up/build-cleaner.spec.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/test/clean-up/build-cleaner.spec.ts index df9b7e074a..cb18ba14be 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/test/clean-up/build-cleaner.spec.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/test/clean-up/build-cleaner.spec.ts @@ -1,135 +1,173 @@ // Imports import * as fs from 'fs'; -import * as path from 'path'; +import {normalize} from 'path'; import * as shell from 'shelljs'; import {BuildCleaner} from '../../lib/clean-up/build-cleaner'; import {HIDDEN_DIR_PREFIX} from '../../lib/common/constants'; import {GithubPullRequests} from '../../lib/common/github-pull-requests'; +const EXISTING_BUILDS = [10, 20, 30, 40]; +const EXISTING_DOWNLOADS = [ + 'downloads/10-ABCDEF0-build.zip', + 'downloads/10-1234567-build.zip', + 'downloads/20-ABCDEF0-build.zip', + 'downloads/20-1234567-build.zip', +]; +const OPEN_PRS = [10, 40]; + // Tests describe('BuildCleaner', () => { let cleaner: BuildCleaner; - beforeEach(() => cleaner = new BuildCleaner('/foo/bar', 'baz/qux', '12345')); + beforeEach(() => cleaner = new BuildCleaner('/foo/bar', 'baz', 'qux', '12345', 'downloads', 'build.zip')); describe('constructor()', () => { it('should throw if \'buildsDir\' is empty', () => { - expect(() => new BuildCleaner('', '/baz/qux', '12345')). + expect(() => new BuildCleaner('', 'baz', 'qux', '12345', 'downloads', 'build.zip')). toThrowError('Missing or empty required parameter \'buildsDir\'!'); }); - it('should throw if \'repoSlug\' is empty', () => { - expect(() => new BuildCleaner('/foo/bar', '', '12345')). - toThrowError('Missing or empty required parameter \'repoSlug\'!'); + it('should throw if \'githubOrg\' is empty', () => { + expect(() => new BuildCleaner('/foo/bar', '', 'qux', '12345', 'downloads', 'build.zip')). + toThrowError('Missing or empty required parameter \'githubOrg\'!'); + }); + + + it('should throw if \'githubRepo\' is empty', () => { + expect(() => new BuildCleaner('/foo/bar', 'baz', '', '12345', 'downloads', 'build.zip')). + toThrowError('Missing or empty required parameter \'githubRepo\'!'); }); it('should throw if \'githubToken\' is empty', () => { - expect(() => new BuildCleaner('/foo/bar', 'baz/qux', '')). + expect(() => new BuildCleaner('/foo/bar', 'baz', 'qux', '', 'downloads', 'build.zip')). toThrowError('Missing or empty required parameter \'githubToken\'!'); }); + it('should throw if \'downloadsDir\' is empty', () => { + expect(() => new BuildCleaner('/foo/bar', 'baz', 'qux', '12345', '', 'build.zip')). + toThrowError('Missing or empty required parameter \'downloadsDir\'!'); + }); + + it('should throw if \'artifactPath\' is empty', () => { + expect(() => new BuildCleaner('/foo/bar', 'baz', 'qux', '12345', 'downloads', '')). + toThrowError('Missing or empty required parameter \'artifactPath\'!'); + }); + }); describe('cleanUp()', () => { let cleanerGetExistingBuildNumbersSpy: jasmine.Spy; let cleanerGetOpenPrNumbersSpy: jasmine.Spy; + let cleanerGetExistingDownloadsSpy: jasmine.Spy; let cleanerRemoveUnnecessaryBuildsSpy: jasmine.Spy; - let existingBuildsDeferred: {resolve: (v?: any) => void, reject: (e?: any) => void}; - let openPrsDeferred: {resolve: (v?: any) => void, reject: (e?: any) => void}; - let promise: Promise; + let cleanerRemoveUnnecessaryDownloadsSpy: jasmine.Spy; beforeEach(() => { - cleanerGetExistingBuildNumbersSpy = spyOn(cleaner as any, 'getExistingBuildNumbers').and.callFake(() => { - return new Promise((resolve, reject) => existingBuildsDeferred = {resolve, reject}); - }); - cleanerGetOpenPrNumbersSpy = spyOn(cleaner as any, 'getOpenPrNumbers').and.callFake(() => { - return new Promise((resolve, reject) => openPrsDeferred = {resolve, reject}); - }); - cleanerRemoveUnnecessaryBuildsSpy = spyOn(cleaner as any, 'removeUnnecessaryBuilds'); + cleanerGetExistingBuildNumbersSpy = spyOn(cleaner, 'getExistingBuildNumbers') + .and.callFake(() => Promise.resolve(EXISTING_BUILDS)); + cleanerGetOpenPrNumbersSpy = spyOn(cleaner, 'getOpenPrNumbers') + .and.callFake(() => Promise.resolve(OPEN_PRS)); + cleanerGetExistingDownloadsSpy = spyOn(cleaner, 'getExistingDownloads') + .and.callFake(() => Promise.resolve(EXISTING_DOWNLOADS)); - promise = cleaner.cleanUp(); + cleanerRemoveUnnecessaryBuildsSpy = spyOn(cleaner, 'removeUnnecessaryBuilds'); + cleanerRemoveUnnecessaryDownloadsSpy = spyOn(cleaner, 'removeUnnecessaryDownloads'); + + spyOn(console, 'log'); }); it('should return a promise', () => { + const promise = cleaner.cleanUp(); expect(promise).toEqual(jasmine.any(Promise)); }); - it('should get the existing builds', () => { - expect(cleanerGetExistingBuildNumbersSpy).toHaveBeenCalled(); - }); - - - it('should get the open PRs', () => { + it('should get the open PRs', async () => { + await cleaner.cleanUp(); expect(cleanerGetOpenPrNumbersSpy).toHaveBeenCalled(); }); - it('should reject if \'getExistingBuildNumbers()\' rejects', done => { - promise.catch(err => { + it('should get the existing builds', async () => { + await cleaner.cleanUp(); + expect(cleanerGetExistingBuildNumbersSpy).toHaveBeenCalled(); + }); + + + it('should get the existing downloads', async () => { + await cleaner.cleanUp(); + expect(cleanerGetExistingDownloadsSpy).toHaveBeenCalled(); + }); + + + it('should pass existing builds and open PRs to \'removeUnnecessaryBuilds()\'', async () => { + await cleaner.cleanUp(); + expect(cleanerRemoveUnnecessaryBuildsSpy).toHaveBeenCalledWith(EXISTING_BUILDS, OPEN_PRS); + }); + + + it('should pass existing downloads and open PRs to \'removeUnnecessaryDownloads()\'', async () => { + await cleaner.cleanUp(); + expect(cleanerRemoveUnnecessaryDownloadsSpy).toHaveBeenCalledWith(EXISTING_DOWNLOADS, OPEN_PRS); + }); + + + it('should reject if \'getOpenPrNumbers()\' rejects', async () => { + try { + cleanerGetOpenPrNumbersSpy.and.callFake(() => Promise.reject('Test')); + await cleaner.cleanUp(); + } catch (err) { expect(err).toBe('Test'); - done(); - }); - - existingBuildsDeferred.reject('Test'); + } }); - it('should reject if \'getOpenPrNumbers()\' rejects', done => { - promise.catch(err => { + it('should reject if \'getExistingBuildNumbers()\' rejects', async () => { + try { + cleanerGetExistingBuildNumbersSpy.and.callFake(() => Promise.reject('Test')); + await cleaner.cleanUp(); + } catch (err) { expect(err).toBe('Test'); - done(); - }); - - openPrsDeferred.reject('Test'); + } }); - it('should reject if \'removeUnnecessaryBuilds()\' rejects', done => { - promise.catch(err => { + it('should reject if \'getExistingDownloads()\' rejects', async () => { + try { + cleanerGetExistingDownloadsSpy.and.callFake(() => Promise.reject('Test')); + await cleaner.cleanUp(); + } catch (err) { expect(err).toBe('Test'); - done(); - }); - - cleanerRemoveUnnecessaryBuildsSpy.and.returnValue(Promise.reject('Test')); - existingBuildsDeferred.resolve(); - openPrsDeferred.resolve(); + } }); - it('should pass existing builds and open PRs to \'removeUnnecessaryBuilds()\'', done => { - promise.then(() => { - expect(cleanerRemoveUnnecessaryBuildsSpy).toHaveBeenCalledWith('foo', 'bar'); - done(); - }); - - existingBuildsDeferred.resolve('foo'); - openPrsDeferred.resolve('bar'); + it('should reject if \'removeUnnecessaryBuilds()\' rejects', async () => { + try { + cleanerRemoveUnnecessaryBuildsSpy.and.callFake(() => Promise.reject('Test')); + await cleaner.cleanUp(); + } catch (err) { + expect(err).toBe('Test'); + } }); - - it('should resolve with the value returned by \'removeUnnecessaryBuilds()\'', done => { - promise.then(result => { - expect(result as any).toBe('Test'); - done(); - }); - - cleanerRemoveUnnecessaryBuildsSpy.and.returnValue(Promise.resolve('Test')); - existingBuildsDeferred.resolve(); - openPrsDeferred.resolve(); + it('should reject if \'removeUnnecessaryDownloads()\' rejects', async () => { + try { + cleanerRemoveUnnecessaryDownloadsSpy.and.callFake(() => Promise.reject('Test')); + await cleaner.cleanUp(); + } catch (err) { + expect(err).toBe('Test'); + } }); - }); - // Protected methods - describe('getExistingBuildNumbers()', () => { let fsReaddirSpy: jasmine.Spy; let readdirCb: (err: any, files?: string[]) => void; @@ -137,7 +175,7 @@ describe('BuildCleaner', () => { beforeEach(() => { fsReaddirSpy = spyOn(fs, 'readdir').and.callFake((_: string, cb: typeof readdirCb) => readdirCb = cb); - promise = (cleaner as any).getExistingBuildNumbers(); + promise = cleaner.getExistingBuildNumbers(); }); @@ -203,7 +241,7 @@ describe('BuildCleaner', () => { return new Promise((resolve, reject) => prDeferred = {resolve, reject}); }); - promise = (cleaner as any).getOpenPrNumbers(); + promise = cleaner.getOpenPrNumbers(); }); @@ -236,6 +274,65 @@ describe('BuildCleaner', () => { prDeferred.resolve([{id: 0, number: 1}, {id: 1, number: 2}, {id: 2, number: 3}]); }); + it('should log the number of open PRs', () => { + promise.then(prNumbers => { + expect(console.log).toHaveBeenCalledWith(`Open pull requests: ${prNumbers}`); + }); + }); + }); + + + describe('getExistingDownloads()', () => { + let fsReaddirSpy: jasmine.Spy; + let readdirCb: (err: any, files?: string[]) => void; + let promise: Promise; + + beforeEach(() => { + fsReaddirSpy = spyOn(fs, 'readdir').and.callFake((_: string, cb: typeof readdirCb) => readdirCb = cb); + promise = cleaner.getExistingDownloads(); + }); + + + it('should return a promise', () => { + expect(promise).toEqual(jasmine.any(Promise)); + }); + + + it('should get the contents of the builds directory', () => { + expect(fsReaddirSpy).toHaveBeenCalled(); + expect(fsReaddirSpy.calls.argsFor(0)[0]).toBe('downloads'); + }); + + + it('should reject if an error occurs while getting the files', done => { + promise.catch(err => { + expect(err).toBe('Test'); + done(); + }); + + readdirCb('Test'); + }); + + + it('should resolve with the returned files (as numbers)', done => { + promise.then(result => { + expect(result).toEqual(EXISTING_DOWNLOADS); + done(); + }); + + readdirCb(null, EXISTING_DOWNLOADS); + }); + + + it('should ignore files that do not match the artifactPath', done => { + promise.then(result => { + expect(result).toEqual(['10-ABCDEF-build.zip', '30-FFFFFFF-build.zip']); + done(); + }); + + readdirCb(null, ['10-ABCDEF-build.zip', '20-AAAAAAA-otherfile.zip', '30-FFFFFFF-build.zip']); + }); + }); @@ -253,7 +350,7 @@ describe('BuildCleaner', () => { it('should test if the directory exists (and return if is does not)', () => { shellTestSpy.and.returnValue(false); - (cleaner as any).removeDir('/foo/bar'); + cleaner.removeDir('/foo/bar'); expect(shellTestSpy).toHaveBeenCalledWith('-d', '/foo/bar'); expect(shellChmodSpy).not.toHaveBeenCalled(); @@ -262,14 +359,14 @@ describe('BuildCleaner', () => { it('should remove the specified directory and its content', () => { - (cleaner as any).removeDir('/foo/bar'); + cleaner.removeDir('/foo/bar'); expect(shellRmSpy).toHaveBeenCalledWith('-rf', '/foo/bar'); }); it('should make the directory and its content writable before removing', () => { shellRmSpy.and.callFake(() => expect(shellChmodSpy).toHaveBeenCalledWith('-R', 'a+w', '/foo/bar')); - (cleaner as any).removeDir('/foo/bar'); + cleaner.removeDir('/foo/bar'); expect(shellRmSpy).toHaveBeenCalled(); }); @@ -282,7 +379,7 @@ describe('BuildCleaner', () => { throw 'Test'; }); - (cleaner as any).removeDir('/foo/bar'); + cleaner.removeDir('/foo/bar'); expect(consoleErrorSpy).toHaveBeenCalled(); expect(consoleErrorSpy.calls.argsFor(0)[0]).toContain('Unable to remove \'/foo/bar\''); @@ -293,68 +390,90 @@ describe('BuildCleaner', () => { describe('removeUnnecessaryBuilds()', () => { - let consoleLogSpy: jasmine.Spy; let cleanerRemoveDirSpy: jasmine.Spy; beforeEach(() => { - consoleLogSpy = spyOn(console, 'log'); - cleanerRemoveDirSpy = spyOn(cleaner as any, 'removeDir'); + spyOn(console, 'log'); + cleanerRemoveDirSpy = spyOn(cleaner, 'removeDir'); }); - it('should log the number of existing builds, open PRs and builds to be removed', () => { - (cleaner as any).removeUnnecessaryBuilds([1, 2, 3], [3, 4, 5, 6]); + it('should log the number of existing builds and builds to be removed', () => { + cleaner.removeUnnecessaryBuilds([1, 2, 3], [3, 4, 5, 6]); expect(console.log).toHaveBeenCalledWith('Existing builds: 3'); - expect(console.log).toHaveBeenCalledWith('Open pull requests: 4'); expect(console.log).toHaveBeenCalledWith('Removing 2 build(s): 1, 2'); }); it('should construct full paths to directories (by prepending \'buildsDir\')', () => { - (cleaner as any).removeUnnecessaryBuilds([1, 2, 3], []); + cleaner.removeUnnecessaryBuilds([1, 2, 3], []); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize('/foo/bar/1')); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize('/foo/bar/2')); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize('/foo/bar/3')); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize('/foo/bar/1')); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize('/foo/bar/2')); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize('/foo/bar/3')); }); it('should try removing hidden directories as well', () => { - (cleaner as any).removeUnnecessaryBuilds([1, 2, 3], []); + cleaner.removeUnnecessaryBuilds([1, 2, 3], []); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}1`)); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}2`)); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}3`)); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}1`)); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}2`)); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}3`)); }); it('should remove the builds that do not correspond to open PRs', () => { - (cleaner as any).removeUnnecessaryBuilds([1, 2, 3, 4], [2, 4]); + cleaner.removeUnnecessaryBuilds([1, 2, 3, 4], [2, 4]); expect(cleanerRemoveDirSpy).toHaveBeenCalledTimes(4); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize('/foo/bar/1')); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize('/foo/bar/3')); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}1`)); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}3`)); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize('/foo/bar/1')); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize('/foo/bar/3')); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}1`)); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}3`)); cleanerRemoveDirSpy.calls.reset(); - (cleaner as any).removeUnnecessaryBuilds([1, 2, 3, 4], [1, 2, 3, 4]); + cleaner.removeUnnecessaryBuilds([1, 2, 3, 4], [1, 2, 3, 4]); expect(cleanerRemoveDirSpy).toHaveBeenCalledTimes(0); cleanerRemoveDirSpy.calls.reset(); (cleaner as any).removeUnnecessaryBuilds([1, 2, 3, 4], []); expect(cleanerRemoveDirSpy).toHaveBeenCalledTimes(8); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize('/foo/bar/1')); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize('/foo/bar/2')); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize('/foo/bar/3')); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize('/foo/bar/4')); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}1`)); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}2`)); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}3`)); - expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(path.normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}4`)); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize('/foo/bar/1')); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize('/foo/bar/2')); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize('/foo/bar/3')); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize('/foo/bar/4')); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}1`)); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}2`)); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}3`)); + expect(cleanerRemoveDirSpy).toHaveBeenCalledWith(normalize(`/foo/bar/${HIDDEN_DIR_PREFIX}4`)); cleanerRemoveDirSpy.calls.reset(); }); }); + + describe('removeUnnecessaryDownloads()', () => { + beforeEach(() => { + spyOn(console, 'log'); + spyOn(shell, 'rm'); + }); + + + it('should remove the downloads that do not correspond to open PRs', () => { + cleaner.removeUnnecessaryDownloads(EXISTING_DOWNLOADS, OPEN_PRS); + expect(shell.rm).toHaveBeenCalledTimes(2); + expect(shell.rm).toHaveBeenCalledWith('downloads/20-ABCDEF0-build.zip'); + expect(shell.rm).toHaveBeenCalledWith('downloads/20-1234567-build.zip'); + }); + + + it('should log the number of existing builds and builds to be removed', () => { + cleaner.removeUnnecessaryDownloads(EXISTING_DOWNLOADS, OPEN_PRS); + + expect(console.log).toHaveBeenCalledWith('Existing downloads: 4'); + expect(console.log).toHaveBeenCalledWith( + 'Removing 2 download(s): downloads/20-ABCDEF0-build.zip, downloads/20-1234567-build.zip'); + }); + }); }); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/circleci-api.spec.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/circleci-api.spec.ts new file mode 100644 index 0000000000..7bd3b5b820 --- /dev/null +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/circleci-api.spec.ts @@ -0,0 +1,134 @@ +import * as nock from 'nock'; +import {CircleCiApi} from '../../lib/common/circle-ci-api'; + +const ORG = 'testorg'; +const REPO = 'testrepo'; +const TOKEN = 'xxxx'; +const BASE_URL = `https://circleci.com/api/v1.1/project/github/${ORG}/${REPO}`; + +describe('CircleCIApi', () => { + describe('constructor()', () => { + it('should throw if \'githubOrg\' is missing or empty', () => { + expect(() => new CircleCiApi('', REPO, TOKEN)). + toThrowError('Missing or empty required parameter \'githubOrg\'!'); + }); + + it('should throw if \'githubRepo\' is missing or empty', () => { + expect(() => new CircleCiApi(ORG, '', TOKEN)). + toThrowError('Missing or empty required parameter \'githubRepo\'!'); + }); + + it('should throw if \'circleCiToken\' is missing or empty', () => { + expect(() => new CircleCiApi(ORG, REPO, '')). + toThrowError('Missing or empty required parameter \'circleCiToken\'!'); + }); + }); + + describe('getBuildInfo', () => { + it('should make a request to the CircleCI API for the given build number', async () => { + const api = new CircleCiApi(ORG, REPO, TOKEN); + const buildNum = 12345; + const expectedBuildInfo: any = { org: ORG, repo: REPO, build_num: buildNum }; + + const request = nock(BASE_URL) + .get(`/${buildNum}?circle-token=${TOKEN}`) + .reply(200, expectedBuildInfo); + + const buildInfo = await api.getBuildInfo(buildNum); + expect(buildInfo).toEqual(expectedBuildInfo); + request.done(); + }); + + it('should throw an error if the request fails', async () => { + const api = new CircleCiApi(ORG, REPO, TOKEN); + const buildNum = 12345; + const errorMessage = 'Invalid request'; + const request = nock(BASE_URL).get(`/${buildNum}?circle-token=${TOKEN}`); + + try { + request.replyWithError(errorMessage); + await api.getBuildInfo(buildNum); + throw new Error('Exception Expected'); + } catch (err) { + expect(err.message).toEqual( + `CircleCI build info request failed ` + + `(request to ${BASE_URL}/${buildNum}?circle-token=${TOKEN} failed, reason: ${errorMessage})`); + } + + try { + request.reply(404, errorMessage); + await api.getBuildInfo(buildNum); + throw new Error('Exception Expected'); + } catch (err) { + expect(err.message).toEqual( + `CircleCI build info request failed ` + + `(request to ${BASE_URL}/${buildNum}?circle-token=${TOKEN} failed, reason: ${errorMessage})`); + } + }); + }); + + describe('getBuildArtifactUrl', () => { + it('should make a request to the CircleCI API for the given build number', async () => { + const api = new CircleCiApi(ORG, REPO, TOKEN); + const buildNum = 12345; + const artifact0: any = { path: 'some/path/0', url: 'https://url/0' }; + const artifact1: any = { path: 'some/path/1', url: 'https://url/1' }; + const artifact2: any = { path: 'some/path/2', url: 'https://url/2' }; + const request = nock(BASE_URL) + .get(`/${buildNum}/artifacts?circle-token=${TOKEN}`) + .reply(200, [artifact0, artifact1, artifact2]); + + const artifactUrl = await api.getBuildArtifactUrl(buildNum, 'some/path/1'); + expect(artifactUrl).toEqual('https://url/1'); + request.done(); + }); + + + it('should throw an error if the request fails', async () => { + const api = new CircleCiApi(ORG, REPO, TOKEN); + const buildNum = 12345; + const errorMessage = 'Invalid request'; + const request = nock(BASE_URL).get(`/${buildNum}/artifacts?circle-token=${TOKEN}`); + + try { + request.replyWithError(errorMessage); + await api.getBuildArtifactUrl(buildNum, 'some/path/1'); + throw new Error('Exception Expected'); + } catch (err) { + expect(err.message).toEqual( + `CircleCI artifact URL request failed ` + + `(request to ${BASE_URL}/${buildNum}/artifacts?circle-token=${TOKEN} failed, reason: ${errorMessage})`); + } + + try { + request.reply(404, errorMessage); + await api.getBuildArtifactUrl(buildNum, 'some/path/1'); + throw new Error('Exception Expected'); + } catch (err) { + expect(err.message).toEqual( + `CircleCI artifact URL request failed ` + + `(request to ${BASE_URL}/${buildNum}/artifacts?circle-token=${TOKEN} failed, reason: ${errorMessage})`); + } + }); + + it('should throw an error if the response does not contain the specified artifact', async () => { + const api = new CircleCiApi(ORG, REPO, TOKEN); + const buildNum = 12345; + const artifact0: any = { path: 'some/path/0', url: 'https://url/0' }; + const artifact1: any = { path: 'some/path/1', url: 'https://url/1' }; + const artifact2: any = { path: 'some/path/2', url: 'https://url/2' }; + nock(BASE_URL) + .get(`/${buildNum}/artifacts?circle-token=${TOKEN}`) + .reply(200, [artifact0, artifact1, artifact2]); + + try { + await api.getBuildArtifactUrl(buildNum, 'some/path/3'); + throw new Error('Exception Expected'); + } catch (err) { + expect(err.message).toEqual( + `CircleCI artifact URL request failed ` + + `(Missing artifact (some/path/3) for CircleCI build: ${buildNum})`); + } + }); + }); +}); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/github-api.spec.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/github-api.spec.ts index 65ff54f61a..7d5250cce4 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/github-api.spec.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/github-api.spec.ts @@ -1,7 +1,5 @@ // Imports -import {EventEmitter} from 'events'; -import {ClientRequest, IncomingMessage} from 'http'; -import * as https from 'https'; +import * as nock from 'nock'; import {GithubApi} from '../../lib/common/github-api'; // Tests @@ -110,39 +108,6 @@ describe('GithubApi', () => { }); - // Protected methods - - describe('buildPath()', () => { - - it('should return the pathname if no params', () => { - expect((api as any).buildPath('/foo')).toBe('/foo'); - expect((api as any).buildPath('/foo', undefined)).toBe('/foo'); - expect((api as any).buildPath('/foo', null)).toBe('/foo'); - }); - - - it('should append the params to the pathname', () => { - expect((api as any).buildPath('/foo', {bar: 'baz'})).toBe('/foo?bar=baz'); - }); - - - it('should join the params with \'&\'', () => { - expect((api as any).buildPath('/foo', {bar: 1, baz: 2})).toBe('/foo?bar=1&baz=2'); - }); - - - it('should ignore undefined/null params', () => { - expect((api as any).buildPath('/foo', {bar: undefined, baz: null})).toBe('/foo'); - }); - - - it('should encode param values as URI components', () => { - expect((api as any).buildPath('/foo', {bar: 'b a&z'})).toBe('/foo?bar=b%20a%26z'); - }); - - }); - - describe('getPaginated()', () => { let deferreds: {resolve: (v: any) => void, reject: (v: any) => void}[]; @@ -218,191 +183,162 @@ describe('GithubApi', () => { }); - describe('request()', () => { - let httpsRequestSpy: jasmine.Spy; - let latestRequest: ClientRequest; + // Protected methods - beforeEach(() => { - const originalRequest = https.request; + describe('buildPath()', () => { - httpsRequestSpy = spyOn(https, 'request').and.callFake((...args: any[]) => { - latestRequest = originalRequest.apply(https, args); - - spyOn(latestRequest, 'on').and.callThrough(); - spyOn(latestRequest, 'end'); - - return latestRequest; - }); + it('should return the pathname if no params', () => { + expect((api as any).buildPath('/foo')).toBe('/foo'); + expect((api as any).buildPath('/foo', undefined)).toBe('/foo'); + expect((api as any).buildPath('/foo', null)).toBe('/foo'); }); + it('should append the params to the pathname', () => { + expect((api as any).buildPath('/foo', {bar: 'baz'})).toBe('/foo?bar=baz'); + }); + + + it('should join the params with \'&\'', () => { + expect((api as any).buildPath('/foo', {bar: 1, baz: 2})).toBe('/foo?bar=1&baz=2'); + }); + + + it('should ignore undefined/null params', () => { + expect((api as any).buildPath('/foo', {bar: undefined, baz: null})).toBe('/foo'); + }); + + + it('should encode param values as URI components', () => { + expect((api as any).buildPath('/foo', {bar: 'b a&z'})).toBe('/foo?bar=b%20a%26z'); + }); + + }); + + describe('request()', () => { it('should return a promise', () => { + nock('https://api.github.com').get('').reply(200); expect((api as any).request()).toEqual(jasmine.any(Promise)); }); it('should call \'https.request()\' with the correct options', () => { - (api as any).request('method', 'path'); + const requestHandler = nock('https://api.github.com') + .intercept('/path', 'method') + .reply(200); - expect(httpsRequestSpy).toHaveBeenCalled(); - expect(httpsRequestSpy.calls.argsFor(0)[0]).toEqual(jasmine.objectContaining({ - headers: jasmine.objectContaining({ - 'User-Agent': `Node/${process.versions.node}`, - }), - host: 'api.github.com', - method: 'method', - path: 'path', - })); + (api as any).request('method', '/path'); + requestHandler.done(); }); - it('should call specify an \'Authorization\' header if \'githubToken\' is present', () => { - (api as any).request('method', 'path'); - - expect(httpsRequestSpy).toHaveBeenCalled(); - expect(httpsRequestSpy.calls.argsFor(0)[0].headers).toEqual(jasmine.objectContaining({ - Authorization: 'token 12345', - })); + it('should add the \'Authorization\' header containing the \'githubToken\'', () => { + const requestHandler = nock('https://api.github.com') + .intercept('/path', 'method', undefined, { + reqheaders: {Authorization: 'token 12345'}, + }) + .reply(200); + (api as any).request('method', '/path'); + requestHandler.done(); }); - it('should reject on request error', done => { - (api as any).request('method', 'path').catch((err: any) => { - expect(err).toBe('Test'); - done(); - }); - - latestRequest.emit('error', 'Test'); - }); - - - it('should send the request (i.e. call \'end()\')', () => { - (api as any).request('method', 'path'); - expect(latestRequest.end).toHaveBeenCalled(); + it('should reject on request error', async () => { + nock('https://api.github.com') + .intercept('/path', 'method') + .replyWithError('Test'); + let message = 'Failed to reject error'; + await (api as any).request('method', '/path').catch((err: any) => message = err.message); + expect(message).toEqual('Test'); }); it('should \'JSON.stringify\' and send the data along with the request', () => { - (api as any).request('method', 'path'); - expect(latestRequest.end).toHaveBeenCalledWith(null); - - (api as any).request('method', 'path', {key: 'value'}); - expect(latestRequest.end).toHaveBeenCalledWith('{"key":"value"}'); + const data = {key: 'value'}; + const requestHandler = nock('https://api.github.com') + .intercept('/path', 'method', JSON.stringify(data)) + .reply(200); + (api as any).request('method', '/path', data); + requestHandler.done(); }); - describe('onResponse', () => { - let promise: Promise; - let respond: (statusCode: number) => IncomingMessage; + it('should reject if response statusCode is <200', done => { + const requestHandler = nock('https://api.github.com') + .intercept('/path', 'method') + .reply(199); - beforeEach(() => { - promise = (api as any).request('method', 'path'); - - respond = (statusCode: number) => { - const mockResponse = new EventEmitter() as IncomingMessage; - mockResponse.statusCode = statusCode; - - const onResponse = httpsRequestSpy.calls.argsFor(0)[1]; - onResponse(mockResponse); - - return mockResponse; - }; - }); - - - it('should reject on response error', done => { - promise.catch(err => { - expect(err).toBe('Test'); - done(); - }); - - const res = respond(200); - res.emit('error', 'Test'); - }); - - - it('should reject if returned statusCode is <200', done => { - promise.catch(err => { + (api as any).request('method', '/path') + .catch((err: string) => { expect(err).toContain('failed'); expect(err).toContain('status: 199'); done(); }); - - const res = respond(199); - res.emit('end'); - }); + requestHandler.done(); + }); - it('should reject if returned statusCode is >=400', done => { - promise.catch(err => { + it('should reject if response statusCode is >=400', done => { + const requestHandler = nock('https://api.github.com') + .intercept('/path', 'method') + .reply(400); + + (api as any).request('method', '/path') + .catch((err: string) => { expect(err).toContain('failed'); expect(err).toContain('status: 400'); done(); }); - - const res = respond(400); - res.emit('end'); - }); + requestHandler.done(); + }); - it('should include the response text in the rejection message', done => { - promise.catch(err => { + it('should include the response text in the rejection message', done => { + const requestHandler = nock('https://api.github.com') + .intercept('/path', 'method') + .reply(500, 'Test'); + + (api as any).request('method', '/path') + .catch((err: string) => { expect(err).toContain('Test'); done(); }); + requestHandler.done(); + }); - const res = respond(500); - res.emit('data', 'Test'); - res.emit('end'); + + it('should resolve if returned statusCode is >=200 and <400', done => { + const requestHandler = nock('https://api.github.com') + .intercept('/path', 'method') + .reply(200); + + (api as any).request('method', '/path').then(done); + requestHandler.done(); + }); + + + it('should parse the response body into an object using \'JSON.parse\'', done => { + const requestHandler = nock('https://api.github.com') + .intercept('/path', 'method') + .reply(300, '{"foo": "bar"}'); + + (api as any).request('method', '/path').then((data: any) => { + expect(data).toEqual({foo: 'bar'}); + done(); }); + requestHandler.done(); + }); + it('should reject if the response text is malformed JSON', done => { + const requestHandler = nock('https://api.github.com') + .intercept('/path', 'method') + .reply(300, '}'); - it('should resolve if returned statusCode is <=200 <400', done => { - promise.then(done); - - const res = respond(200); - res.emit('data', '{}'); - res.emit('end'); + (api as any).request('method', '/path').catch((err: any) => { + expect(err).toEqual(jasmine.any(SyntaxError)); + done(); }); - - - it('should resolve with the response text \'JSON.parsed\'', done => { - promise.then(data => { - expect(data).toEqual({foo: 'bar'}); - done(); - }); - - const res = respond(300); - res.emit('data', '{"foo":"bar"}'); - res.emit('end'); - }); - - - it('should collect and concatenate the whole response text', done => { - promise.then(data => { - expect(data).toEqual({foo: 'bar', baz: 'qux'}); - done(); - }); - - const res = respond(300); - res.emit('data', '{"foo":'); - res.emit('data', '"bar","baz"'); - res.emit('data', ':"qux"}'); - res.emit('end'); - }); - - - it('should reject if the response text is malformed JSON', done => { - promise.catch(err => { - expect(err).toEqual(jasmine.any(SyntaxError)); - done(); - }); - - const res = respond(300); - res.emit('data', '}'); - res.emit('end'); - }); - + requestHandler.done(); }); }); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/github-pull-requests.spec.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/github-pull-requests.spec.ts index db0f70dd03..39e4c07793 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/github-pull-requests.spec.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/github-pull-requests.spec.ts @@ -1,20 +1,27 @@ // Imports +import {GithubApi} from '../../lib/common/github-api'; import {GithubPullRequests} from '../../lib/common/github-pull-requests'; // Tests describe('GithubPullRequests', () => { + let githubApi: jasmine.SpyObj; + + beforeEach(() => { + githubApi = jasmine.createSpyObj('githubApi', ['post', 'get', 'getPaginated']); + }); + describe('constructor()', () => { - it('should throw if \'githubToken\' is missing or empty', () => { - expect(() => new GithubPullRequests('', 'foo/bar')). - toThrowError('Missing or empty required parameter \'githubToken\'!'); + it('should throw if \'githubOrg\' is missing or empty', () => { + expect(() => new GithubPullRequests(githubApi, '', 'bar')). + toThrowError('Missing or empty required parameter \'githubOrg\'!'); }); - it('should throw if \'repoSlug\' is missing or empty', () => { - expect(() => new GithubPullRequests('12345', '')). - toThrowError('Missing or empty required parameter \'repoSlug\'!'); + it('should throw if \'githubRepo\' is missing or empty', () => { + expect(() => new GithubPullRequests(githubApi, 'foo', '')). + toThrowError('Missing or empty required parameter \'githubRepo\'!'); }); }); @@ -22,17 +29,9 @@ describe('GithubPullRequests', () => { describe('addComment()', () => { let prs: GithubPullRequests; - let deferred: {resolve: (v: any) => void, reject: (v: any) => void}; beforeEach(() => { - prs = new GithubPullRequests('12345', 'foo/bar'); - - spyOn(prs, 'post').and.callFake(() => new Promise((resolve, reject) => deferred = {resolve, reject})); - }); - - - it('should return a promise', () => { - expect(prs.addComment(42, 'body')).toEqual(jasmine.any(Promise)); + prs = new GithubPullRequests(githubApi, 'foo', 'bar'); }); @@ -47,30 +46,28 @@ describe('GithubPullRequests', () => { }); - it('should call \'post()\' with the correct pathname, params and data', () => { + it('should make a POST request to Github with the correct pathname, params and data', () => { + githubApi.post.and.callFake(() => Promise.resolve()); prs.addComment(42, 'body'); - - expect(prs.post).toHaveBeenCalledWith('/repos/foo/bar/issues/42/comments', null, {body: 'body'}); + expect(githubApi.post).toHaveBeenCalledWith('/repos/foo/bar/issues/42/comments', null, {body: 'body'}); }); it('should reject if the request fails', done => { + githubApi.post.and.callFake(() => Promise.reject('Test')); prs.addComment(42, 'body').catch(err => { expect(err).toBe('Test'); done(); }); - - deferred.reject('Test'); }); - it('should resolve with the returned response', done => { + it('should resolve with the data from the Github POST', done => { + githubApi.post.and.callFake(() => Promise.resolve('Test')); prs.addComment(42, 'body').then(data => { - expect(data as any).toBe('Test'); + expect(data).toBe('Test'); done(); }); - - deferred.resolve('Test'); }); }); @@ -78,35 +75,34 @@ describe('GithubPullRequests', () => { describe('fetch()', () => { let prs: GithubPullRequests; - let prsGetSpy: jasmine.Spy; beforeEach(() => { - prs = new GithubPullRequests('12345', 'foo/bar'); - prsGetSpy = spyOn(prs as any, 'get'); + prs = new GithubPullRequests(githubApi, 'foo', 'bar'); }); - it('should call \'get()\' with the correct pathname', () => { + it('should make a GET request to GitHub with the correct pathname', () => { prs.fetch(42); - expect(prsGetSpy).toHaveBeenCalledWith('/repos/foo/bar/issues/42'); + expect(githubApi.get).toHaveBeenCalledWith('/repos/foo/bar/issues/42'); }); - it('should forward the value returned by \'get()\'', () => { - prsGetSpy.and.returnValue('Test'); - expect(prs.fetch(42) as any).toBe('Test'); + it('should resolve with the data returned from GitHub', done => { + const expected: any = {number: 42}; + githubApi.get.and.callFake(() => Promise.resolve(expected)); + prs.fetch(42).then(data => { + expect(data).toEqual(expected); + done(); + }); }); - }); describe('fetchAll()', () => { let prs: GithubPullRequests; - let prsGetPaginatedSpy: jasmine.Spy; beforeEach(() => { - prs = new GithubPullRequests('12345', 'foo/bar'); - prsGetPaginatedSpy = spyOn(prs as any, 'getPaginated'); + prs = new GithubPullRequests(githubApi, 'foo', 'bar'); spyOn(console, 'log'); }); @@ -118,24 +114,48 @@ describe('GithubPullRequests', () => { prs.fetchAll('closed'); prs.fetchAll('open'); - expect(prsGetPaginatedSpy).toHaveBeenCalledTimes(3); - expect(prsGetPaginatedSpy.calls.argsFor(0)).toEqual([expectedPathname, {state: 'all'}]); - expect(prsGetPaginatedSpy.calls.argsFor(1)).toEqual([expectedPathname, {state: 'closed'}]); - expect(prsGetPaginatedSpy.calls.argsFor(2)).toEqual([expectedPathname, {state: 'open'}]); + expect(githubApi.getPaginated).toHaveBeenCalledTimes(3); + expect(githubApi.getPaginated.calls.argsFor(0)).toEqual([expectedPathname, {state: 'all'}]); + expect(githubApi.getPaginated.calls.argsFor(1)).toEqual([expectedPathname, {state: 'closed'}]); + expect(githubApi.getPaginated.calls.argsFor(2)).toEqual([expectedPathname, {state: 'open'}]); }); it('should default to \'all\' if no state is specified', () => { prs.fetchAll(); - expect(prsGetPaginatedSpy).toHaveBeenCalledWith('/repos/foo/bar/pulls', {state: 'all'}); + expect(githubApi.getPaginated).toHaveBeenCalledWith('/repos/foo/bar/pulls', {state: 'all'}); }); it('should forward the value returned by \'getPaginated()\'', () => { - prsGetPaginatedSpy.and.returnValue('Test'); + githubApi.getPaginated.and.returnValue('Test'); expect(prs.fetchAll() as any).toBe('Test'); }); - }); + describe('fetchFiles()', () => { + let prs: GithubPullRequests; + + beforeEach(() => { + prs = new GithubPullRequests(githubApi, 'foo', 'bar'); + }); + + + it('should make a GET request to GitHub with the correct pathname', () => { + prs.fetchFiles(42); + expect(githubApi.get).toHaveBeenCalledWith('/repos/foo/bar/pulls/42/files'); + }); + + + it('should resolve with the data returned from GitHub', done => { + const expected: any = [{ sha: 'ABCDE', filename: 'a/b/c'}, { sha: '12345', filename: 'x/y/z' }]; + githubApi.get.and.callFake(() => Promise.resolve(expected)); + prs.fetch(42).then(data => { + expect(data).toEqual(expected); + done(); + }); + }); + }); + + }); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/github-teams.spec.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/github-teams.spec.ts index a9b03517c0..2a089c92b9 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/github-teams.spec.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/github-teams.spec.ts @@ -1,43 +1,40 @@ -// Imports +import {GithubApi} from '../../lib/common/github-api'; import {GithubTeams} from '../../lib/common/github-teams'; // Tests describe('GithubTeams', () => { + let githubApi: jasmine.SpyObj; + + beforeEach(() => { + githubApi = jasmine.createSpyObj('githubApi', ['post', 'get', 'getPaginated']); + }); + describe('constructor()', () => { - it('should throw if \'githubToken\' is missing or empty', () => { - expect(() => new GithubTeams('', 'org')). - toThrowError('Missing or empty required parameter \'githubToken\'!'); + it('should throw if \'githubOrg\' is missing or empty', () => { + expect(() => new GithubTeams(githubApi, '')). + toThrowError('Missing or empty required parameter \'githubOrg\'!'); }); - - - it('should throw if \'organization\' is missing or empty', () => { - expect(() => new GithubTeams('12345', '')). - toThrowError('Missing or empty required parameter \'organization\'!'); - }); - }); describe('fetchAll()', () => { let teams: GithubTeams; - let teamsGetPaginatedSpy: jasmine.Spy; beforeEach(() => { - teams = new GithubTeams('12345', 'foo'); - teamsGetPaginatedSpy = spyOn(teams as any, 'getPaginated'); + teams = new GithubTeams(githubApi, 'foo'); }); it('should call \'getPaginated()\' with the correct pathname and params', () => { teams.fetchAll(); - expect(teamsGetPaginatedSpy).toHaveBeenCalledWith('/orgs/foo/teams'); + expect(githubApi.getPaginated).toHaveBeenCalledWith('/orgs/foo/teams'); }); it('should forward the value returned by \'getPaginated()\'', () => { - teamsGetPaginatedSpy.and.returnValue('Test'); + githubApi.getPaginated.and.returnValue('Test'); expect(teams.fetchAll() as any).toBe('Test'); }); @@ -46,19 +43,15 @@ describe('GithubTeams', () => { describe('isMemberById()', () => { let teams: GithubTeams; - let teamsGetSpy: jasmine.Spy; beforeEach(() => { - teams = new GithubTeams('12345', 'foo'); - teamsGetSpy = spyOn(teams, 'get').and.returnValue(Promise.resolve(null)); + teams = new GithubTeams(githubApi, 'foo'); }); - it('should return a promise', done => { + it('should return a promise', () => { + githubApi.get.and.callFake(() => Promise.resolve()); const promise = teams.isMemberById('user', [1]); - promise.then(done); // Do not complete the test (and release the spies) synchronously - // to avoid running the actual `get()`. - expect(promise).toEqual(jasmine.any(Promise)); }); @@ -66,42 +59,43 @@ describe('GithubTeams', () => { it('should resolve with false if called with an empty array', done => { teams.isMemberById('user', []).then(isMember => { expect(isMember).toBe(false); - expect(teamsGetSpy).not.toHaveBeenCalled(); + expect(githubApi.get).not.toHaveBeenCalled(); done(); }); }); it('should call \'get()\' with the correct pathname', done => { + githubApi.get.and.callFake(() => Promise.resolve()); teams.isMemberById('user', [1]).then(() => { - expect(teamsGetSpy).toHaveBeenCalledWith('/teams/1/memberships/user'); + expect(githubApi.get).toHaveBeenCalledWith('/teams/1/memberships/user'); done(); }); }); it('should resolve with false if \'get()\' rejects', done => { - teamsGetSpy.and.returnValue(Promise.reject(null)); + githubApi.get.and.callFake(() => Promise.reject(null)); teams.isMemberById('user', [1]).then(isMember => { expect(isMember).toBe(false); - expect(teamsGetSpy).toHaveBeenCalled(); + expect(githubApi.get).toHaveBeenCalled(); done(); }); }); it('should resolve with false if the membership is not active', done => { - teamsGetSpy.and.returnValue(Promise.resolve({state: 'pending'})); + githubApi.get.and.callFake(() => Promise.resolve({state: 'pending'})); teams.isMemberById('user', [1]).then(isMember => { expect(isMember).toBe(false); - expect(teamsGetSpy).toHaveBeenCalled(); + expect(githubApi.get).toHaveBeenCalled(); done(); }); }); it('should resolve with true if the membership is active', done => { - teamsGetSpy.and.returnValue(Promise.resolve({state: 'active'})); + githubApi.get.and.callFake(() => Promise.resolve({state: 'active'})); teams.isMemberById('user', [1]).then(isMember => { expect(isMember).toBe(true); done(); @@ -115,15 +109,15 @@ describe('GithubTeams', () => { '/teams/2/memberships/user': Promise.reject(null), '/teams/3/memberships/user': Promise.resolve({state: 'active'}), }; - teamsGetSpy.and.callFake((pathname: string) => trainedResponses[pathname]); + githubApi.get.and.callFake((pathname: string) => trainedResponses[pathname]); teams.isMemberById('user', [1, 2, 3, 4]).then(isMember => { expect(isMember).toBe(true); - expect(teamsGetSpy).toHaveBeenCalledTimes(3); - expect(teamsGetSpy.calls.argsFor(0)[0]).toBe('/teams/1/memberships/user'); - expect(teamsGetSpy.calls.argsFor(1)[0]).toBe('/teams/2/memberships/user'); - expect(teamsGetSpy.calls.argsFor(2)[0]).toBe('/teams/3/memberships/user'); + expect(githubApi.get).toHaveBeenCalledTimes(3); + expect(githubApi.get.calls.argsFor(0)[0]).toBe('/teams/1/memberships/user'); + expect(githubApi.get.calls.argsFor(1)[0]).toBe('/teams/2/memberships/user'); + expect(githubApi.get.calls.argsFor(2)[0]).toBe('/teams/3/memberships/user'); done(); }); @@ -137,16 +131,16 @@ describe('GithubTeams', () => { '/teams/3/memberships/user': Promise.resolve({state: 'not active'}), '/teams/4/memberships/user': Promise.reject(null), }; - teamsGetSpy.and.callFake((pathname: string) => trainedResponses[pathname]); + githubApi.get.and.callFake((pathname: string) => trainedResponses[pathname]); teams.isMemberById('user', [1, 2, 3, 4]).then(isMember => { expect(isMember).toBe(false); - expect(teamsGetSpy).toHaveBeenCalledTimes(4); - expect(teamsGetSpy.calls.argsFor(0)[0]).toBe('/teams/1/memberships/user'); - expect(teamsGetSpy.calls.argsFor(1)[0]).toBe('/teams/2/memberships/user'); - expect(teamsGetSpy.calls.argsFor(2)[0]).toBe('/teams/3/memberships/user'); - expect(teamsGetSpy.calls.argsFor(3)[0]).toBe('/teams/4/memberships/user'); + expect(githubApi.get).toHaveBeenCalledTimes(4); + expect(githubApi.get.calls.argsFor(0)[0]).toBe('/teams/1/memberships/user'); + expect(githubApi.get.calls.argsFor(1)[0]).toBe('/teams/2/memberships/user'); + expect(githubApi.get.calls.argsFor(2)[0]).toBe('/teams/3/memberships/user'); + expect(githubApi.get.calls.argsFor(3)[0]).toBe('/teams/4/memberships/user'); done(); }); @@ -161,7 +155,7 @@ describe('GithubTeams', () => { let teamsIsMemberByIdSpy: jasmine.Spy; beforeEach(() => { - teams = new GithubTeams('12345', 'foo'); + teams = new GithubTeams(githubApi, 'foo'); const mockResponse = Promise.resolve([{id: 1, slug: 'team1'}, {id: 2, slug: 'team2'}]); teamsFetchAllSpy = spyOn(teams, 'fetchAll').and.returnValue(mockResponse); @@ -181,7 +175,7 @@ describe('GithubTeams', () => { it('should resolve with false if \'fetchAll()\' rejects', done => { - teamsFetchAllSpy.and.returnValue(Promise.reject(null)); + teamsFetchAllSpy.and.callFake(() => Promise.reject(null)); teams.isMemberBySlug('user', ['team-slug']).then(isMember => { expect(isMember).toBe(false); done(); @@ -209,7 +203,7 @@ describe('GithubTeams', () => { it('should resolve with false if \'isMemberById()\' rejects', done => { - teamsIsMemberByIdSpy.and.returnValue(Promise.reject(null)); + teamsIsMemberByIdSpy.and.callFake(() => Promise.reject(null)); teams.isMemberBySlug('user', ['team1']).then(isMember => { expect(isMember).toBe(false); expect(teamsIsMemberByIdSpy).toHaveBeenCalled(); @@ -218,16 +212,17 @@ describe('GithubTeams', () => { }); - it('should resolve with the value \'isMemberById()\' resolves with', done => { - teamsIsMemberByIdSpy.and.returnValues(Promise.resolve(false), Promise.resolve(true)); + it('should resolve with the value \'isMemberById()\' resolves with', async () => { - Promise.all([ - teams.isMemberBySlug('user', ['team1']).then(isMember => expect(isMember).toBe(false)), - teams.isMemberBySlug('user', ['team1']).then(isMember => expect(isMember).toBe(true)), - ]).then(() => { - expect(teamsIsMemberByIdSpy).toHaveBeenCalledTimes(2); - done(); - }); + teamsIsMemberByIdSpy.and.callFake(() => Promise.resolve(true)); + const isMember1 = await teams.isMemberBySlug('user', ['team1']); + expect(isMember1).toBe(true); + expect(teamsIsMemberByIdSpy).toHaveBeenCalledWith('user', [1]); + + teamsIsMemberByIdSpy.and.callFake(() => Promise.resolve(false)); + const isMember2 = await teams.isMemberBySlug('user', ['team1']); + expect(isMember2).toBe(false); + expect(teamsIsMemberByIdSpy).toHaveBeenCalledWith('user', [1]); }); }); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/utils.spec.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/utils.spec.ts index 872fa2fdd8..160b2d5b71 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/utils.spec.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/test/common/utils.spec.ts @@ -1,9 +1,53 @@ // Imports -import {assertNotMissingOrEmpty, getEnvVar} from '../../lib/common/utils'; +import { + assert, + assertNotMissingOrEmpty, + computeArtifactDownloadPath, + computeShortSha, + getEnvVar, + getPrInfoFromDownloadPath, +} from '../../lib/common/utils'; // Tests describe('utils', () => { + describe('computeShortSha', () => { + it('should return only the first SHORT_SHA_LEN characters of the SHA', () => { + expect(computeShortSha('0123456789')).toEqual('0123456'); + expect(computeShortSha('ABC')).toEqual('ABC'); + expect(computeShortSha('')).toEqual(''); + }); + }); + + describe('assert', () => { + it('should throw if passed a false value', () => { + expect(() => assert(false, 'error message')).toThrowError('error message'); + }); + + it('should not throw if passed a true value', () => { + expect(() => assert(true, 'error message')).not.toThrow(); + }); + }); + + describe('computeArtifactDownloadPath', () => { + it('should compute an absolute path based on the artifact info provided', () => { + const downloadDir = '/a/b/c'; + const pr = 123; + const sha = 'ABCDEF1234567'; + const artifactPath = 'a/path/to/file.zip'; + const path = computeArtifactDownloadPath(downloadDir, pr, sha, artifactPath); + expect(path).toEqual('/a/b/c/123-ABCDEF1-file.zip'); + }); + }); + + describe('getPrInfoFromDownloadPath', () => { + it('should extract the PR and SHA from the file path', () => { + const {pr, sha} = getPrInfoFromDownloadPath('a/b/c/12345-ABCDE-artifact.zip'); + expect(pr).toEqual(12345); + expect(sha).toEqual('ABCDE'); + }); + }); + describe('assertNotMissingOrEmpty()', () => { it('should throw if passed an empty value', () => { diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/build-creator.spec.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/build-creator.spec.ts index 469beb73a0..bc167feab5 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/build-creator.spec.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/build-creator.spec.ts @@ -12,13 +12,13 @@ import {expectToBeUploadError} from './helpers'; // Tests describe('BuildCreator', () => { - const pr = '9'; + const pr = 9; const sha = '9'.repeat(40); const shortSha = sha.substr(0, SHORT_SHA_LEN); const archive = 'snapshot.tar.gz'; const buildsDir = 'builds/dir'; const hiddenPrDir = path.join(buildsDir, `hidden--${pr}`); - const publicPrDir = path.join(buildsDir, pr); + const publicPrDir = path.join(buildsDir, `${pr}`); const hiddenShaDir = path.join(hiddenPrDir, shortSha); const publicShaDir = path.join(publicPrDir, shortSha); let bc: BuildCreator; @@ -135,7 +135,7 @@ describe('BuildCreator', () => { it('should abort and skip further operations if changing the PR\'s visibility fails', done => { const mockError = new UploadError(543, 'Test'); - bcUpdatePrVisibilitySpy.and.returnValue(Promise.reject(mockError)); + bcUpdatePrVisibilitySpy.and.callFake(() => Promise.reject(mockError)); bc.create(pr, sha, archive, isPublic).catch(err => { expect(err).toBe(mockError); @@ -324,7 +324,7 @@ describe('BuildCreator', () => { const shas = ['foo', 'bar', 'baz']; let emitted = false; - bcListShasByDate.and.returnValue(Promise.resolve(shas)); + bcListShasByDate.and.callFake(() => Promise.resolve(shas)); bcEmitSpy.and.callFake((type: string, evt: ChangedPrVisibilityEvent) => { expect(bcListShasByDate).toHaveBeenCalledWith(newPrDir); @@ -451,7 +451,7 @@ describe('BuildCreator', () => { it('should call \'fs.access()\' with the specified argument', () => { (bc as any).exists('foo'); - expect(fs.access).toHaveBeenCalledWith('foo', jasmine.any(Function)); + expect(fsAccessSpy).toHaveBeenCalledWith('foo', jasmine.any(Function)); }); @@ -618,7 +618,7 @@ describe('BuildCreator', () => { it('should reject if listing files fails', done => { - shellLsSpy.and.returnValue(Promise.reject('Test')); + shellLsSpy.and.callFake(() => Promise.reject('Test')); (bc as any).listShasByDate('input/dir').catch((err: string) => { expect(err).toBe('Test'); done(); @@ -627,7 +627,7 @@ describe('BuildCreator', () => { it('should return the filenames', done => { - shellLsSpy.and.returnValue(Promise.resolve([ + shellLsSpy.and.callFake(() => Promise.resolve([ lsResult('foo', 100), lsResult('bar', 200), lsResult('baz', 300), @@ -640,7 +640,7 @@ describe('BuildCreator', () => { it('should sort by date', done => { - shellLsSpy.and.returnValue(Promise.resolve([ + shellLsSpy.and.callFake(() => Promise.resolve([ lsResult('foo', 300), lsResult('bar', 100), lsResult('baz', 200), @@ -660,7 +660,7 @@ describe('BuildCreator', () => { ]; mockArray.sort = jasmine.createSpy('sort'); - shellLsSpy.and.returnValue(Promise.resolve(mockArray)); + shellLsSpy.and.callFake(() => Promise.resolve(mockArray)); (bc as any).listShasByDate('input/dir'). then((shas: string[]) => { expect(shas).toEqual(['bar', 'baz', 'foo']); @@ -671,7 +671,7 @@ describe('BuildCreator', () => { it('should only include directories', done => { - shellLsSpy.and.returnValue(Promise.resolve([ + shellLsSpy.and.callFake(() => Promise.resolve([ lsResult('foo', 100), lsResult('bar', 200, false), lsResult('baz', 300), diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/build-retriever.spec.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/build-retriever.spec.ts new file mode 100644 index 0000000000..ab11ca8ca1 --- /dev/null +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/build-retriever.spec.ts @@ -0,0 +1,191 @@ +import * as fs from 'fs'; +import * as nock from 'nock'; +import {BuildInfo, CircleCiApi} from '../../lib/common/circle-ci-api'; +import {BuildRetriever} from '../../lib/upload-server/build-retriever'; + +describe('BuildRetriever', () => { + const MAX_DOWNLOAD_SIZE = 10000; + const DOWNLOAD_DIR = '/DOWNLOAD/DIR'; + const BASE_URL = 'http://test.com'; + const ARTIFACT_PATH = '/some/path/build.zip'; + + let api: CircleCiApi; + let BUILD_INFO: BuildInfo; + let WRITEFILE_RESULT: any; + let writeFileSpy: jasmine.Spy; + let EXISTS_RESULT: boolean; + let existsSpy: jasmine.Spy; + let getBuildArtifactUrlSpy: jasmine.Spy; + + beforeEach(() => { + BUILD_INFO = { + branch: 'pull/777', + build_num: 12345, + failed: false, + has_artifacts: true, + outcome: 'success', + reponame: 'REPO', + username: 'ORG', + vcs_revision: 'COMMIT', + }; + + spyOn(console, 'log'); + spyOn(console, 'warn'); + spyOn(console, 'error'); + + api = new CircleCiApi('ORG', 'REPO', 'TOKEN'); + spyOn(api, 'getBuildInfo').and.callFake(() => Promise.resolve(BUILD_INFO)); + getBuildArtifactUrlSpy = spyOn(api, 'getBuildArtifactUrl') + .and.callFake(() => Promise.resolve(BASE_URL + ARTIFACT_PATH)); + + WRITEFILE_RESULT = undefined; + writeFileSpy = spyOn(fs, 'writeFile').and.callFake( + (_path: string, _buffer: Buffer, callback: (err?: any) => {}) => callback(WRITEFILE_RESULT), + ); + + EXISTS_RESULT = false; + existsSpy = spyOn(fs, 'exists').and.callFake( + (_path: string, callback: (exists: boolean) => {}) => callback(EXISTS_RESULT), + ); + }); + + describe('constructor', () => { + it('should fail if the "downloadSizeLimit" is invalid', () => { + expect(() => new BuildRetriever(api, NaN, DOWNLOAD_DIR)) + .toThrowError(`Invalid parameter "downloadSizeLimit" should be a number greater than 0.`); + expect(() => new BuildRetriever(api, 0, DOWNLOAD_DIR)) + .toThrowError(`Invalid parameter "downloadSizeLimit" should be a number greater than 0.`); + expect(() => new BuildRetriever(api, -1, DOWNLOAD_DIR)) + .toThrowError(`Invalid parameter "downloadSizeLimit" should be a number greater than 0.`); + }); + it('should fail if the "downloadDir" is missing', () => { + expect(() => new BuildRetriever(api, MAX_DOWNLOAD_SIZE, '')) + .toThrowError(`Missing or empty required parameter 'downloadDir'!`); + }); + }); + + + describe('getGithubInfo', () => { + it('should request the info from CircleCI', async () => { + const retriever = new BuildRetriever(api, MAX_DOWNLOAD_SIZE, DOWNLOAD_DIR); + const info = await retriever.getGithubInfo(12345); + expect(api.getBuildInfo).toHaveBeenCalledWith(12345); + expect(info).toEqual({org: 'ORG', pr: 777, repo: 'REPO', sha: 'COMMIT', success: true}); + }); + + it('should error if it is not possible to extract the PR number from the branch', async () => { + const retriever = new BuildRetriever(api, MAX_DOWNLOAD_SIZE, DOWNLOAD_DIR); + try { + BUILD_INFO.branch = 'master'; + await retriever.getGithubInfo(12345); + throw new Error('Exception Expected'); + } catch (error) { + expect(error.message).toEqual('No PR found in branch field: master'); + } + }); + }); + + + describe('downloadBuildArtifact', () => { + const ARTIFACT_CONTENTS = 'ARTIFACT CONTENTS'; + let retriever: BuildRetriever; + + beforeEach(() => { + retriever = new BuildRetriever(api, MAX_DOWNLOAD_SIZE, DOWNLOAD_DIR); + }); + + it('should get the artifact URL from the CircleCI API', async () => { + const artifactRequest = nock(BASE_URL).get(ARTIFACT_PATH).reply(200, ARTIFACT_CONTENTS); + await retriever.downloadBuildArtifact(12345, 777, 'COMMIT', ARTIFACT_PATH); + expect(api.getBuildArtifactUrl).toHaveBeenCalledWith(12345, ARTIFACT_PATH); + artifactRequest.done(); + }); + + it('should download the artifact from its URL', async () => { + const artifactRequest = nock(BASE_URL).get(ARTIFACT_PATH).reply(200, ARTIFACT_CONTENTS); + await retriever.downloadBuildArtifact(12345, 777, 'COMMIT', ARTIFACT_PATH); + // The following line proves that the artifact URL fetch occurred. + artifactRequest.done(); + }); + + it('should fail if the artifact is too large', async () => { + const artifactRequest = nock(BASE_URL).get(ARTIFACT_PATH).reply(200, ARTIFACT_CONTENTS); + retriever = new BuildRetriever(api, 10, DOWNLOAD_DIR); + try { + await retriever.downloadBuildArtifact(12345, 777, 'COMMIT', ARTIFACT_PATH); + throw new Error('Exception Expected'); + } catch (error) { + expect(error.status).toEqual(413); + } + artifactRequest.done(); + }); + + it('should not download the artifact if it already exists', async () => { + const artifactRequestInterceptor = nock(BASE_URL).get(ARTIFACT_PATH); + const artifactRequest = artifactRequestInterceptor.reply(200, ARTIFACT_CONTENTS); + EXISTS_RESULT = true; + await retriever.downloadBuildArtifact(12345, 777, 'COMMIT', ARTIFACT_PATH); + expect(existsSpy).toHaveBeenCalled(); + expect(getBuildArtifactUrlSpy).not.toHaveBeenCalled(); + expect(artifactRequest.isDone()).toEqual(false); + nock.removeInterceptor(artifactRequestInterceptor); + }); + + it('should write the artifact file to disk', async () => { + const artifactRequest = nock(BASE_URL).get(ARTIFACT_PATH).reply(200, ARTIFACT_CONTENTS); + await retriever.downloadBuildArtifact(12345, 777, 'COMMIT', ARTIFACT_PATH); + expect(writeFileSpy) + .toHaveBeenCalledWith(`${DOWNLOAD_DIR}/777-COMMIT-build.zip`, jasmine.any(Buffer), jasmine.any(Function)); + const buffer: Buffer = writeFileSpy.calls.mostRecent().args[1]; + expect(buffer.toString()).toEqual(ARTIFACT_CONTENTS); + artifactRequest.done(); + }); + + it('should fail if the CircleCI API fails', async () => { + try { + getBuildArtifactUrlSpy.and.callFake(() => Promise.reject('getBuildArtifactUrl failed')); + await retriever.downloadBuildArtifact(12345, 777, 'COMMIT', ARTIFACT_PATH); + throw new Error('Exception Expected'); + } catch (error) { + expect(error.message).toEqual('CircleCI artifact download failed (getBuildArtifactUrl failed)'); + } + }); + + it('should fail if the URL fetch errors', async () => { + // create a new handler that errors + const artifactRequest = nock(BASE_URL).get(ARTIFACT_PATH).replyWithError('Artifact Request Failed'); + try { + await retriever.downloadBuildArtifact(12345, 777, 'COMMIT', ARTIFACT_PATH); + throw new Error('Exception Expected'); + } catch (error) { + expect(error.message).toEqual('CircleCI artifact download failed ' + + '(request to http://test.com/some/path/build.zip failed, reason: Artifact Request Failed)'); + } + artifactRequest.done(); + }); + + it('should fail if the URL fetch 404s', async () => { + // create a new handler that errors + const artifactRequest = nock(BASE_URL).get(ARTIFACT_PATH).reply(404, 'No such artifact'); + try { + await retriever.downloadBuildArtifact(12345, 777, 'COMMIT', ARTIFACT_PATH); + throw new Error('Exception Expected'); + } catch (error) { + expect(error.message).toEqual('CircleCI artifact download failed (Error 404 - Not Found)'); + } + artifactRequest.done(); + }); + + it('should fail if file write fails', async () => { + const artifactRequest = nock(BASE_URL).get(ARTIFACT_PATH).reply(200, ARTIFACT_CONTENTS); + try { + WRITEFILE_RESULT = 'Test Error'; + await retriever.downloadBuildArtifact(12345, 777, 'COMMIT', ARTIFACT_PATH); + throw new Error('Exception Expected'); + } catch (error) { + expect(error.message).toEqual('CircleCI artifact download failed (Test Error)'); + } + artifactRequest.done(); + }); + }); +}); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/build-verifier.spec.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/build-verifier.spec.ts index aa8f501aa3..1e84cf1715 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/build-verifier.spec.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/build-verifier.spec.ts @@ -1,27 +1,29 @@ // Imports -import * as jwt from 'jsonwebtoken'; +import {GithubApi} from '../../lib/common/github-api'; import {GithubPullRequests, PullRequest} from '../../lib/common/github-pull-requests'; import {GithubTeams} from '../../lib/common/github-teams'; -import {BUILD_VERIFICATION_STATUS, BuildVerifier} from '../../lib/upload-server/build-verifier'; -import {expectToBeUploadError} from './helpers'; +import {BuildVerifier} from '../../lib/upload-server/build-verifier'; // Tests describe('BuildVerifier', () => { const defaultConfig = { allowedTeamSlugs: ['team1', 'team2'], + githubOrg: 'organization', + githubRepo: 'repo', githubToken: 'githubToken', - organization: 'organization', - repoSlug: 'repo/slug', secret: 'secret', trustedPrLabel: 'trusted: pr-label', }; + let prs: GithubPullRequests; let bv: BuildVerifier; // Helpers const createBuildVerifier = (partialConfig: Partial = {}) => { const cfg = {...defaultConfig, ...partialConfig} as typeof defaultConfig; - return new BuildVerifier(cfg.secret, cfg.githubToken, cfg.repoSlug, cfg.organization, - cfg.allowedTeamSlugs, cfg.trustedPrLabel); + const api = new GithubApi(cfg.githubToken); + prs = new GithubPullRequests(api, cfg.githubOrg, cfg.githubRepo); + const teams = new GithubTeams(api, cfg.githubOrg); + return new BuildVerifier(prs, teams, cfg.allowedTeamSlugs, cfg.trustedPrLabel); }; beforeEach(() => bv = createBuildVerifier()); @@ -29,7 +31,7 @@ describe('BuildVerifier', () => { describe('constructor()', () => { - ['secret', 'githubToken', 'repoSlug', 'organization', 'allowedTeamSlugs', 'trustedPrLabel']. + ['githubToken', 'githubRepo', 'githubOrg', 'allowedTeamSlugs', 'trustedPrLabel']. forEach(param => { it(`should throw if '${param}' is missing or empty`, () => { expect(() => createBuildVerifier({[param]: ''})). @@ -46,6 +48,20 @@ describe('BuildVerifier', () => { }); + describe('getSignificantFilesChanged', () => { + it('should return false if none of the fetched files match the given pattern', async () => { + const fetchFilesSpy = spyOn(prs, 'fetchFiles'); + fetchFilesSpy.and.callFake(() => Promise.resolve([{filename: 'a/b/c'}, {filename: 'd/e/f'}])); + expect(await bv.getSignificantFilesChanged(777, /^x/)).toEqual(false); + expect(fetchFilesSpy).toHaveBeenCalledWith(777); + + fetchFilesSpy.calls.reset(); + expect(await bv.getSignificantFilesChanged(777, /^a/)).toEqual(true); + expect(fetchFilesSpy).toHaveBeenCalledWith(777); + }); + }); + + describe('getPrIsTrusted()', () => { const pr = 9; let mockPrInfo: PullRequest; @@ -63,10 +79,10 @@ describe('BuildVerifier', () => { }; prsFetchSpy = spyOn(GithubPullRequests.prototype, 'fetch'). - and.returnValue(Promise.resolve(mockPrInfo)); + and.callFake(() => Promise.resolve(mockPrInfo)); teamsIsMemberBySlugSpy = spyOn(GithubTeams.prototype, 'isMemberBySlug'). - and.returnValue(Promise.resolve(true)); + and.callFake(() => Promise.resolve(true)); }); @@ -139,7 +155,7 @@ describe('BuildVerifier', () => { it('should resolve to true if the PR\'s author is a member', done => { - teamsIsMemberBySlugSpy.and.returnValue(Promise.resolve(true)); + teamsIsMemberBySlugSpy.and.callFake(() => Promise.resolve(true)); bv.getPrIsTrusted(pr).then(isTrusted => { expect(isTrusted).toBe(true); @@ -149,7 +165,7 @@ describe('BuildVerifier', () => { it('should resolve to false if the PR\'s author is not a member', done => { - teamsIsMemberBySlugSpy.and.returnValue(Promise.resolve(false)); + teamsIsMemberBySlugSpy.and.callFake(() => Promise.resolve(false)); bv.getPrIsTrusted(pr).then(isTrusted => { expect(isTrusted).toBe(false); @@ -161,143 +177,4 @@ describe('BuildVerifier', () => { }); - - describe('verify()', () => { - const pr = 9; - const defaultJwt = { - 'exp': Math.floor(Date.now() / 1000) + 30, - 'iat': Math.floor(Date.now() / 1000) - 30, - 'iss': 'Travis CI, GmbH', - 'pull-request': pr, - 'slug': defaultConfig.repoSlug, - }; - let bvGetPrIsTrusted: jasmine.Spy; - - // Heleprs - const createAuthHeader = (partialJwt: Partial = {}, secret: string = defaultConfig.secret) => - `Token ${jwt.sign({...defaultJwt, ...partialJwt}, secret)}`; - - beforeEach(() => { - bvGetPrIsTrusted = spyOn(bv, 'getPrIsTrusted').and.returnValue(Promise.resolve(true)); - }); - - - it('should return a promise', done => { - const promise = bv.verify(pr, createAuthHeader()); - promise.then(done); // Do not complete the test (and release the spies) synchronously - // to avoid running the actual `bvGetPrIsTrusted()`. - - expect(promise).toEqual(jasmine.any(Promise)); - }); - - - it('should fail if the authorization header is invalid', done => { - bv.verify(pr, 'foo').catch(err => { - const errorMessage = 'Error while verifying upload for PR 9: jwt malformed'; - - expectToBeUploadError(err, 403, errorMessage); - done(); - }); - }); - - - it('should fail if the secret is invalid', done => { - bv.verify(pr, createAuthHeader({}, 'foo')).catch(err => { - const errorMessage = 'Error while verifying upload for PR 9: invalid signature'; - - expectToBeUploadError(err, 403, errorMessage); - done(); - }); - }); - - - it('should fail if the issuer is invalid', done => { - bv.verify(pr, createAuthHeader({iss: 'not valid'})).catch(err => { - const errorMessage = 'Error while verifying upload for PR 9: ' + - `jwt issuer invalid. expected: ${defaultJwt.iss}`; - - expectToBeUploadError(err, 403, errorMessage); - done(); - }); - }); - - - it('should fail if the token has expired', done => { - bv.verify(pr, createAuthHeader({exp: 0})).catch(err => { - const errorMessage = 'Error while verifying upload for PR 9: jwt expired'; - - expectToBeUploadError(err, 403, errorMessage); - done(); - }); - }); - - - it('should fail if the repo slug does not match', done => { - bv.verify(pr, createAuthHeader({slug: 'foo/bar'})).catch(err => { - const errorMessage = 'Error while verifying upload for PR 9: ' + - `jwt slug invalid. expected: ${defaultConfig.repoSlug}`; - - expectToBeUploadError(err, 403, errorMessage); - done(); - }); - }); - - - it('should fail if the PR does not match', done => { - bv.verify(pr, createAuthHeader({'pull-request': 1337})).catch(err => { - const errorMessage = 'Error while verifying upload for PR 9: ' + - `jwt pull-request invalid. expected: ${pr}`; - - expectToBeUploadError(err, 403, errorMessage); - done(); - }); - }); - - - it('should not fail if the token is valid', done => { - bv.verify(pr, createAuthHeader()).then(done); - }); - - - it('should not fail even if the token has been issued in the future', done => { - const in30s = Math.floor(Date.now() / 1000) + 30; - bv.verify(pr, createAuthHeader({iat: in30s})).then(done); - }); - - - it('should call \'getPrIsTrusted()\' if the token is valid', done => { - bv.verify(pr, createAuthHeader()).then(() => { - expect(bvGetPrIsTrusted).toHaveBeenCalledWith(pr); - done(); - }); - }); - - - it('should fail if \'getPrIsTrusted()\' rejects', done => { - bvGetPrIsTrusted.and.callFake(() => Promise.reject('Test')); - bv.verify(pr, createAuthHeader()).catch(err => { - expectToBeUploadError(err, 403, `Error while verifying upload for PR ${pr}: Test`); - done(); - }); - }); - - - it('should resolve to `verifiedNotTrusted` if \'getPrIsTrusted()\' returns false', done => { - bvGetPrIsTrusted.and.returnValue(Promise.resolve(false)); - bv.verify(pr, createAuthHeader()).then(value => { - expect(value).toBe(BUILD_VERIFICATION_STATUS.verifiedNotTrusted); - done(); - }); - }); - - - it('should resolve to `verifiedAndTrusted` if \'getPrIsTrusted()\' returns true', done => { - bv.verify(pr, createAuthHeader()).then(value => { - expect(value).toBe(BUILD_VERIFICATION_STATUS.verifiedAndTrusted); - done(); - }); - }); - - }); - }); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/upload-server-factory.spec.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/upload-server-factory.spec.ts index d0db5552e4..ea8c29535a 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/upload-server-factory.spec.ts +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/upload-server-factory.spec.ts @@ -2,35 +2,53 @@ import * as express from 'express'; import * as http from 'http'; import * as supertest from 'supertest'; +import {promisify} from 'util'; +import {CircleCiApi} from '../../lib/common/circle-ci-api'; +import {GithubApi} from '../../lib/common/github-api'; import {GithubPullRequests} from '../../lib/common/github-pull-requests'; +import {GithubTeams} from '../../lib/common/github-teams'; import {BuildCreator} from '../../lib/upload-server/build-creator'; import {ChangedPrVisibilityEvent, CreatedBuildEvent} from '../../lib/upload-server/build-events'; -import {BUILD_VERIFICATION_STATUS, BuildVerifier} from '../../lib/upload-server/build-verifier'; -import {uploadServerFactory as usf} from '../../lib/upload-server/upload-server-factory'; +import {BuildRetriever, GithubInfo} from '../../lib/upload-server/build-retriever'; +import {BuildVerifier} from '../../lib/upload-server/build-verifier'; +import {UploadServerConfig, UploadServerFactory} from '../../lib/upload-server/upload-server-factory'; + +interface CircleCiWebHookPayload { + payload: { + build_num: number; + build_parameters: { + CIRCLE_JOB: string; + } + }; +} // Tests describe('uploadServerFactory', () => { - const defaultConfig = { + const defaultConfig: UploadServerConfig = { + buildArtifactPath: 'artifact/path.zip', buildsDir: 'builds/dir', + circleCiToken: 'CIRCLE_CI_TOKEN', domainName: 'domain.name', - githubOrganization: 'organization', + downloadSizeLimit: 999, + downloadsDir: '/tmp/aio-create-builds', + githubOrg: 'organisation', + githubRepo: 'repo', githubTeamSlugs: ['team1', 'team2'], githubToken: '12345', - repoSlug: 'repo/slug', - secret: 'secret', + significantFilesPattern: '^(?:aio|packages)\\/(?!.*[._]spec\\.[jt]s$)', trustedPrLabel: 'trusted: pr-label', }; // Helpers - const createUploadServer = (partialConfig: Partial = {}) => - usf.create({...defaultConfig, ...partialConfig} as typeof defaultConfig); + const createUploadServer = (partialConfig: Partial = {}) => + UploadServerFactory.create({...defaultConfig, ...partialConfig}); describe('create()', () => { let usfCreateMiddlewareSpy: jasmine.Spy; beforeEach(() => { - usfCreateMiddlewareSpy = spyOn(usf as any, 'createMiddleware').and.callThrough(); + usfCreateMiddlewareSpy = spyOn(UploadServerFactory, 'createMiddleware').and.callThrough(); }); @@ -52,9 +70,9 @@ describe('uploadServerFactory', () => { }); - it('should throw if \'githubOrganization\' is missing or empty', () => { - expect(() => createUploadServer({githubOrganization: ''})). - toThrowError('Missing or empty required parameter \'organization\'!'); + it('should throw if \'githubOrg\' is missing or empty', () => { + expect(() => createUploadServer({githubOrg: ''})). + toThrowError('Missing or empty required parameter \'githubOrg\'!'); }); @@ -64,15 +82,9 @@ describe('uploadServerFactory', () => { }); - it('should throw if \'repoSlug\' is missing or empty', () => { - expect(() => createUploadServer({repoSlug: ''})). - toThrowError('Missing or empty required parameter \'repoSlug\'!'); - }); - - - it('should throw if \'secret\' is missing or empty', () => { - expect(() => createUploadServer({secret: ''})). - toThrowError('Missing or empty required parameter \'secret\'!'); + it('should throw if \'githubRepo\' is missing or empty', () => { + expect(() => createUploadServer({githubRepo: ''})). + toThrowError('Missing or empty required parameter \'githubRepo\'!'); }); @@ -91,13 +103,16 @@ describe('uploadServerFactory', () => { it('should create and use an appropriate BuildCreator', () => { - const usfCreateBuildCreatorSpy = spyOn(usf as any, 'createBuildCreator').and.callThrough(); + const usfCreateBuildCreatorSpy = spyOn(UploadServerFactory, 'createBuildCreator').and.callThrough(); createUploadServer(); + const buildRetriever = jasmine.any(BuildRetriever); + const buildVerifier = jasmine.any(BuildVerifier); + const prs = jasmine.any(GithubPullRequests); const buildCreator: BuildCreator = usfCreateBuildCreatorSpy.calls.mostRecent().returnValue; - expect(usfCreateMiddlewareSpy).toHaveBeenCalledWith(jasmine.any(BuildVerifier), buildCreator); - expect(usfCreateBuildCreatorSpy).toHaveBeenCalledWith('builds/dir', '12345', 'repo/slug', 'domain.name'); + expect(usfCreateMiddlewareSpy).toHaveBeenCalledWith(buildRetriever, buildVerifier, buildCreator, defaultConfig); + expect(usfCreateBuildCreatorSpy).toHaveBeenCalledWith(prs, 'builds/dir', 'domain.name'); }); @@ -105,12 +120,14 @@ describe('uploadServerFactory', () => { const httpCreateServerSpy = spyOn(http, 'createServer').and.callThrough(); createUploadServer(); - const middleware: express.Express = usfCreateMiddlewareSpy.calls.mostRecent().returnValue; + + const buildRetriever = jasmine.any(BuildRetriever); const buildVerifier = jasmine.any(BuildVerifier); const buildCreator = jasmine.any(BuildCreator); + expect(usfCreateMiddlewareSpy).toHaveBeenCalledWith(buildRetriever, buildVerifier, buildCreator, defaultConfig); + const middleware: express.Express = usfCreateMiddlewareSpy.calls.mostRecent().returnValue; expect(httpCreateServerSpy).toHaveBeenCalledWith(middleware); - expect(usfCreateMiddlewareSpy).toHaveBeenCalledWith(buildVerifier, buildCreator); }); @@ -134,15 +151,11 @@ describe('uploadServerFactory', () => { let buildCreator: BuildCreator; beforeEach(() => { - buildCreator = (usf as any).createBuildCreator( - defaultConfig.buildsDir, - defaultConfig.githubToken, - defaultConfig.repoSlug, - defaultConfig.domainName, - ); + const api = new GithubApi(defaultConfig.githubToken); + const prs = new GithubPullRequests(api, defaultConfig.githubOrg, defaultConfig.githubRepo); + buildCreator = UploadServerFactory.createBuildCreator(prs, defaultConfig.buildsDir, defaultConfig.domainName); }); - it('should pass the \'buildsDir\' to the BuildCreator', () => { expect((buildCreator as any).buildsDir).toBe('builds/dir'); }); @@ -199,248 +212,241 @@ describe('uploadServerFactory', () => { }); - it('should pass the correct \'githubToken\' and \'repoSlug\' to GithubPullRequests', () => { + it('should pass the correct parameters to GithubPullRequests', () => { const prsAddCommentSpy = spyOn(GithubPullRequests.prototype, 'addComment'); buildCreator.emit(CreatedBuildEvent.type, {pr: 42, sha: '1234567890', isPublic: true}); buildCreator.emit(ChangedPrVisibilityEvent.type, {pr: 42, shas: ['12345', '67890'], isPublic: true}); const allCalls = prsAddCommentSpy.calls.all(); - const prs = allCalls[0].object; + const prs: GithubPullRequests = allCalls[0].object; expect(prsAddCommentSpy).toHaveBeenCalledTimes(2); expect(prs).toBe(allCalls[1].object); expect(prs).toEqual(jasmine.any(GithubPullRequests)); - expect(prs.repoSlug).toBe('repo/slug'); - expect(prs.requestHeaders.Authorization).toContain('12345'); + expect(prs.repoSlug).toBe('organisation/repo'); }); }); describe('createMiddleware()', () => { + let buildRetriever: BuildRetriever; let buildVerifier: BuildVerifier; let buildCreator: BuildCreator; let agent: supertest.SuperTest; // Helpers - const promisifyRequest = (req: supertest.Request) => - new Promise((resolve, reject) => req.end(err => err ? reject(err) : resolve())); - const verifyRequests = (reqs: supertest.Request[], done: jasmine.DoneFn) => - Promise.all(reqs.map(promisifyRequest)).then(done, done.fail); + const promisifyRequest = async (req: supertest.Request) => await promisify(req.end.bind(req))(); + const verifyRequests = async (reqs: supertest.Request[]) => await Promise.all(reqs.map(promisifyRequest)); beforeEach(() => { - buildVerifier = new BuildVerifier( - defaultConfig.secret, - defaultConfig.githubToken, - defaultConfig.repoSlug, - defaultConfig.githubOrganization, - defaultConfig.githubTeamSlugs, - defaultConfig.trustedPrLabel, - ); + const circleCiApi = new CircleCiApi(defaultConfig.githubOrg, defaultConfig.githubRepo, + defaultConfig.circleCiToken); + const githubApi = new GithubApi(defaultConfig.githubToken); + const prs = new GithubPullRequests(githubApi, defaultConfig.githubOrg, defaultConfig.githubRepo); + const teams = new GithubTeams(githubApi, defaultConfig.githubOrg); + + buildRetriever = new BuildRetriever(circleCiApi, defaultConfig.downloadSizeLimit, defaultConfig.downloadsDir); + buildVerifier = new BuildVerifier(prs, teams, defaultConfig.githubTeamSlugs, defaultConfig.trustedPrLabel); buildCreator = new BuildCreator(defaultConfig.buildsDir); - agent = supertest.agent((usf as any).createMiddleware(buildVerifier, buildCreator)); + + const middleware = UploadServerFactory.createMiddleware(buildRetriever, buildVerifier, buildCreator, + defaultConfig); + agent = supertest.agent(middleware); spyOn(console, 'error'); }); - - describe('GET /create-build//', () => { - const pr = '9'; - const sha = '9'.repeat(40); - let buildVerifierVerifySpy: jasmine.Spy; - let buildCreatorCreateSpy: jasmine.Spy; - - beforeEach(() => { - const verStatus = BUILD_VERIFICATION_STATUS.verifiedAndTrusted; - buildVerifierVerifySpy = spyOn(buildVerifier, 'verify').and.returnValue(Promise.resolve(verStatus)); - buildCreatorCreateSpy = spyOn(buildCreator, 'create').and.returnValue(Promise.resolve()); - }); - - - it('should respond with 404 for non-GET requests', done => { - verifyRequests([ - agent.put(`/create-build/${pr}/${sha}`).expect(404), - agent.post(`/create-build/${pr}/${sha}`).expect(404), - agent.patch(`/create-build/${pr}/${sha}`).expect(404), - agent.delete(`/create-build/${pr}/${sha}`).expect(404), - ], done); - }); - - - it('should respond with 401 for requests without an \'AUTHORIZATION\' header', done => { - const url = `/create-build/${pr}/${sha}`; - const responseBody = `Missing or empty 'AUTHORIZATION' header in request: GET ${url}`; - - verifyRequests([ - agent.get(url).expect(401, responseBody), - agent.get(url).set('AUTHORIZATION', '').expect(401, responseBody), - ], done); - }); - - - it('should respond with 400 for requests without an \'X-FILE\' header', done => { - const url = `/create-build/${pr}/${sha}`; - const responseBody = `Missing or empty 'X-FILE' header in request: GET ${url}`; - - const request1 = agent.get(url).set('AUTHORIZATION', 'foo'); - const request2 = agent.get(url).set('AUTHORIZATION', 'foo').set('X-FILE', ''); - - verifyRequests([ - request1.expect(400, responseBody), - request2.expect(400, responseBody), - ], done); - }); - - - it('should respond with 404 for unknown paths', done => { - verifyRequests([ - agent.get(`/foo/create-build/${pr}/${sha}`).expect(404), - agent.get(`/foo-create-build/${pr}/${sha}`).expect(404), - agent.get(`/fooncreate-build/${pr}/${sha}`).expect(404), - agent.get(`/create-build/foo/${pr}/${sha}`).expect(404), - agent.get(`/create-build-foo/${pr}/${sha}`).expect(404), - agent.get(`/create-buildnfoo/${pr}/${sha}`).expect(404), - agent.get(`/create-build/pr${pr}/${sha}`).expect(404), - agent.get(`/create-build/${pr}/${sha}42`).expect(404), - ], done); - }); - - - it('should call \'BuildVerifier#verify()\' with the correct arguments', done => { - const req = agent. - get(`/create-build/${pr}/${sha}`). - set('AUTHORIZATION', 'foo'). - set('X-FILE', 'bar'); - - promisifyRequest(req). - then(() => expect(buildVerifierVerifySpy).toHaveBeenCalledWith(9, 'foo')). - then(done, done.fail); - }); - - - it('should propagate errors from BuildVerifier', done => { - buildVerifierVerifySpy.and.callFake(() => Promise.reject('Test')); - - const req = agent. - get(`/create-build/${pr}/${sha}`). - set('AUTHORIZATION', 'foo'). - set('X-FILE', 'bar'). - expect(500, 'Test'); - - promisifyRequest(req). - then(() => { - expect(buildVerifierVerifySpy).toHaveBeenCalledWith(9, 'foo'); - expect(buildCreatorCreateSpy).not.toHaveBeenCalled(); - }). - then(done, done.fail); - }); - - - it('should call \'BuildCreator#create()\' with the correct arguments', done => { - buildVerifierVerifySpy.and.returnValues( - Promise.resolve(BUILD_VERIFICATION_STATUS.verifiedAndTrusted), - Promise.resolve(BUILD_VERIFICATION_STATUS.verifiedNotTrusted)); - - const req1 = agent.get(`/create-build/${pr}/${sha}`).set('AUTHORIZATION', 'foo').set('X-FILE', 'bar'); - const req2 = agent.get(`/create-build/${pr}/${sha}`).set('AUTHORIZATION', 'foo').set('X-FILE', 'bar'); - - Promise.all([ - promisifyRequest(req1).then(() => expect(buildCreatorCreateSpy).toHaveBeenCalledWith(pr, sha, 'bar', true)), - promisifyRequest(req2).then(() => expect(buildCreatorCreateSpy).toHaveBeenCalledWith(pr, sha, 'bar', false)), - ]).then(done, done.fail); - }); - - - it('should propagate errors from BuildCreator', done => { - buildCreatorCreateSpy.and.callFake(() => Promise.reject('Test')); - const req = agent. - get(`/create-build/${pr}/${sha}`). - set('AUTHORIZATION', 'foo'). - set('X-FILE', 'bar'). - expect(500, 'Test'); - - verifyRequests([req], done); - }); - - - it('should respond with 201 on successful upload (for public builds)', done => { - const req = agent. - get(`/create-build/${pr}/${sha}`). - set('AUTHORIZATION', 'foo'). - set('X-FILE', 'bar'). - expect(201, http.STATUS_CODES[201]); - - verifyRequests([req], done); - }); - - - it('should respond with 202 on successful upload (for hidden builds)', done => { - buildVerifierVerifySpy.and.returnValue(Promise.resolve(BUILD_VERIFICATION_STATUS.verifiedNotTrusted)); - const req = agent. - get(`/create-build/${pr}/${sha}`). - set('AUTHORIZATION', 'foo'). - set('X-FILE', 'bar'). - expect(202, http.STATUS_CODES[202]); - - verifyRequests([req], done); - }); - - - it('should reject PRs with leading zeros', done => { - verifyRequests([agent.get(`/create-build/0${pr}/${sha}`).expect(404)], done); - }); - - - it('should accept SHAs with leading zeros (but not trim the zeros)', done => { - const sha40 = '0'.repeat(40); - const sha41 = `0${sha40}`; - - const request40 = agent.get(`/create-build/${pr}/${sha40}`).set('AUTHORIZATION', 'foo').set('X-FILE', 'bar'); - const request41 = agent.get(`/create-build/${pr}/${sha41}`).set('AUTHORIZATION', 'baz').set('X-FILE', 'qux'); - - Promise.all([ - promisifyRequest(request40.expect(201)), - promisifyRequest(request41.expect(404)), - ]).then(done, done.fail); - }); - - }); - - describe('GET /health-check', () => { - it('should respond with 200', done => { - verifyRequests([ + it('should respond with 200', async () => { + await verifyRequests([ agent.get('/health-check').expect(200), agent.get('/health-check/').expect(200), - ], done); + ]); }); - it('should respond with 404 for non-GET requests', done => { - verifyRequests([ + it('should respond with 404 for non-GET requests', async () => { + await verifyRequests([ agent.put('/health-check').expect(404), agent.post('/health-check').expect(404), agent.patch('/health-check').expect(404), agent.delete('/health-check').expect(404), - ], done); + ]); }); - it('should respond with 404 if the path does not match exactly', done => { - verifyRequests([ + it('should respond with 404 if the path does not match exactly', async () => { + await verifyRequests([ agent.get('/health-check/foo').expect(404), agent.get('/health-check-foo').expect(404), agent.get('/health-checknfoo').expect(404), agent.get('/foo/health-check').expect(404), agent.get('/foo-health-check').expect(404), agent.get('/foonhealth-check').expect(404), - ], done); + ]); }); }); + describe('/circle-build', () => { + let getGithubInfoSpy: jasmine.Spy; + let getSignificantFilesChangedSpy: jasmine.Spy; + let downloadBuildArtifactSpy: jasmine.Spy; + let getPrIsTrustedSpy: jasmine.Spy; + let createBuildSpy: jasmine.Spy; + let IS_PUBLIC: boolean; + let BUILD_INFO: GithubInfo; + let AFFECTS_SIGNIFICANT_FILES: boolean; + let BASIC_PAYLOAD: CircleCiWebHookPayload; + const URL = '/circle-build'; + const BUILD_NUM = 12345; + const PR = 777; + const SHA = 'COMMIT'; + const DOWNLOADED_ARTIFACT_PATH = 'downloads/777-COMMIT-build.zip'; + + beforeEach(() => { + IS_PUBLIC = true; + BUILD_INFO = { + org: defaultConfig.githubOrg, + pr: PR, + repo: defaultConfig.githubRepo, + sha: SHA, + success: true, + }; + BASIC_PAYLOAD = { payload: { build_num: BUILD_NUM, build_parameters: { CIRCLE_JOB: 'aio_preview' } } }; + AFFECTS_SIGNIFICANT_FILES = true; + getGithubInfoSpy = spyOn(buildRetriever, 'getGithubInfo') + .and.callFake(() => Promise.resolve(BUILD_INFO)); + getSignificantFilesChangedSpy = spyOn(buildVerifier, 'getSignificantFilesChanged') + .and.callFake(() => Promise.resolve(AFFECTS_SIGNIFICANT_FILES)); + downloadBuildArtifactSpy = spyOn(buildRetriever, 'downloadBuildArtifact') + .and.callFake(() => Promise.resolve(DOWNLOADED_ARTIFACT_PATH)); + getPrIsTrustedSpy = spyOn(buildVerifier, 'getPrIsTrusted') + .and.callFake(() => Promise.resolve(IS_PUBLIC)); + createBuildSpy = spyOn(buildCreator, 'create'); + }); + + it('should respond with 400 if the request body is not in the correct format', async () => { + await Promise.all([ + agent.post(URL).expect(400), + agent.post(URL).send().expect(400), + agent.post(URL).send({}).expect(400), + agent.post(URL).send({ payload: {} }).expect(400), + agent.post(URL).send({ payload: { build_num: -1 } }).expect(400), + agent.post(URL).send({ payload: { build_num: 4000 } }).expect(400), + agent.post(URL).send({ payload: { build_num: 4000, build_parameters: { } } }).expect(400), + agent.post(URL).send({ payload: { build_num: 4000, build_parameters: { CIRCLE_JOB: '' } } }).expect(400), + ]); + }); + + it('should create a preview if everything is good and the build succeeded', async () => { + await agent.post(URL).send(BASIC_PAYLOAD).expect(201); + expect(getGithubInfoSpy).toHaveBeenCalledWith(BUILD_NUM); + expect(getSignificantFilesChangedSpy).toHaveBeenCalledWith(PR, jasmine.any(RegExp)); + expect(downloadBuildArtifactSpy).toHaveBeenCalledWith(BUILD_NUM, PR, SHA, defaultConfig.buildArtifactPath); + expect(getPrIsTrustedSpy).toHaveBeenCalledWith(PR); + expect(createBuildSpy).toHaveBeenCalledWith(PR, SHA, DOWNLOADED_ARTIFACT_PATH, IS_PUBLIC); + }); + + it('should respond with 204 if the reported build is not the "AIO preview" job', async () => { + BASIC_PAYLOAD.payload.build_parameters.CIRCLE_JOB = 'lint'; + await agent.post(URL).send(BASIC_PAYLOAD).expect(204); + expect(getGithubInfoSpy).not.toHaveBeenCalled(); + expect(getSignificantFilesChangedSpy).not.toHaveBeenCalled(); + expect(console.log).toHaveBeenCalledWith(jasmine.any(String), 'UploadServer: ', + 'Build:12345, Job:lint -', 'Skipping preview processing because this is not the "aio_preview" job.'); + expect(downloadBuildArtifactSpy).not.toHaveBeenCalled(); + expect(getPrIsTrustedSpy).not.toHaveBeenCalled(); + expect(createBuildSpy).not.toHaveBeenCalled(); + }); + + it('should respond with 204 if the build did not affect any significant files', async () => { + spyOn(console, 'log'); + AFFECTS_SIGNIFICANT_FILES = false; + await agent.post(URL).send(BASIC_PAYLOAD).expect(204); + expect(getGithubInfoSpy).toHaveBeenCalledWith(BUILD_NUM); + expect(getSignificantFilesChangedSpy).toHaveBeenCalledWith(PR, jasmine.any(RegExp)); + expect(console.log).toHaveBeenCalledWith( + 'PR:777, Build:12345 - Skipping preview processing because this PR did not touch any significant files.'); + expect(downloadBuildArtifactSpy).not.toHaveBeenCalled(); + expect(getPrIsTrustedSpy).not.toHaveBeenCalled(); + expect(createBuildSpy).not.toHaveBeenCalled(); + }); + + it('should respond with 201 if the build is trusted', async () => { + IS_PUBLIC = true; + await agent.post(URL).send(BASIC_PAYLOAD).expect(201); + }); + + it('should respond with 202 if the build is not trusted', async () => { + IS_PUBLIC = false; + await agent.post(URL).send(BASIC_PAYLOAD).expect(202); + }); + + it('should not create a preview if the build was not successful', async () => { + BUILD_INFO.success = false; + await agent.post(URL).send(BASIC_PAYLOAD).expect(204); + expect(getGithubInfoSpy).toHaveBeenCalledWith(BUILD_NUM); + expect(downloadBuildArtifactSpy).not.toHaveBeenCalled(); + expect(getPrIsTrustedSpy).not.toHaveBeenCalled(); + expect(createBuildSpy).not.toHaveBeenCalled(); + }); + + it('should fail if the CircleCI request fails', async () => { + // Note it is important to put the `reject` into `and.callFake`; + // If you just `and.returnValue` the rejected promise + // then you get an "unhandled rejection" message in the console. + getGithubInfoSpy.and.callFake(() => Promise.reject('Test Error')); + await agent.post(URL).send(BASIC_PAYLOAD).expect(500, 'Test Error'); + expect(getGithubInfoSpy).toHaveBeenCalledWith(BUILD_NUM); + expect(downloadBuildArtifactSpy).not.toHaveBeenCalled(); + expect(getPrIsTrustedSpy).not.toHaveBeenCalled(); + expect(createBuildSpy).not.toHaveBeenCalled(); + }); + + it('should fail if the Github organisation of the build does not match the configured organisation', async () => { + BUILD_INFO.org = 'bad'; + await agent.post(URL).send(BASIC_PAYLOAD) + .expect(500, `Invalid webhook: expected "githubOrg" property to equal "organisation" but got "bad".`); + }); + + it('should fail if the Github repo of the build does not match the configured repo', async () => { + BUILD_INFO.repo = 'bad'; + await agent.post(URL).send(BASIC_PAYLOAD) + .expect(500, `Invalid webhook: expected "githubRepo" property to equal "repo" but got "bad".`); + }); + + it('should fail if the artifact fetch request fails', async () => { + downloadBuildArtifactSpy.and.callFake(() => Promise.reject('Test Error')); + await agent.post(URL).send(BASIC_PAYLOAD).expect(500, 'Test Error'); + expect(getGithubInfoSpy).toHaveBeenCalledWith(BUILD_NUM); + expect(downloadBuildArtifactSpy).toHaveBeenCalled(); + expect(getPrIsTrustedSpy).not.toHaveBeenCalled(); + expect(createBuildSpy).not.toHaveBeenCalled(); + }); + + it('should fail if verifying the PR fails', async () => { + getPrIsTrustedSpy.and.callFake(() => Promise.reject('Test Error')); + await agent.post(URL).send(BASIC_PAYLOAD).expect(500, 'Test Error'); + expect(getGithubInfoSpy).toHaveBeenCalledWith(BUILD_NUM); + expect(downloadBuildArtifactSpy).toHaveBeenCalled(); + expect(getPrIsTrustedSpy).toHaveBeenCalled(); + expect(createBuildSpy).not.toHaveBeenCalled(); + }); + + it('should fail if creating the preview build fails', async () => { + createBuildSpy.and.callFake(() => Promise.reject('Test Error')); + await agent.post(URL).send(BASIC_PAYLOAD).expect(500, 'Test Error'); + expect(getGithubInfoSpy).toHaveBeenCalledWith(BUILD_NUM); + expect(downloadBuildArtifactSpy).toHaveBeenCalled(); + expect(getPrIsTrustedSpy).toHaveBeenCalled(); + expect(createBuildSpy).toHaveBeenCalled(); + }); + }); + describe('POST /pr-updated', () => { const pr = '9'; @@ -458,123 +464,112 @@ describe('uploadServerFactory', () => { }); - it('should respond with 404 for non-POST requests', done => { - verifyRequests([ + it('should respond with 404 for non-POST requests', async () => { + await verifyRequests([ agent.get(url).expect(404), agent.put(url).expect(404), agent.patch(url).expect(404), agent.delete(url).expect(404), - ], done); + ]); }); - it('should respond with 400 for requests without a payload', done => { + it('should respond with 400 for requests without a payload', async () => { const responseBody = `Missing or empty 'number' field in request: POST ${url} {}`; const request1 = agent.post(url); const request2 = agent.post(url).send(); - verifyRequests([ + await verifyRequests([ request1.expect(400, responseBody), request2.expect(400, responseBody), - ], done); + ]); }); - it('should respond with 400 for requests without a \'number\' field', done => { + it('should respond with 400 for requests without a \'number\' field', async () => { const responseBodyPrefix = `Missing or empty 'number' field in request: POST ${url}`; const request1 = agent.post(url).send({}); const request2 = agent.post(url).send({number: null}); - verifyRequests([ + await verifyRequests([ request1.expect(400, `${responseBodyPrefix} {}`), request2.expect(400, `${responseBodyPrefix} {"number":null}`), - ], done); + ]); }); - it('should call \'BuildVerifier#gtPrIsTrusted()\' with the correct arguments', done => { - const req = createRequest(+pr); - - promisifyRequest(req). - then(() => expect(bvGetPrIsTrustedSpy).toHaveBeenCalledWith(9)). - then(done, done.fail); + it('should call \'BuildVerifier#gtPrIsTrusted()\' with the correct arguments', async () => { + await promisifyRequest(createRequest(+pr)); + expect(bvGetPrIsTrustedSpy).toHaveBeenCalledWith(9); }); - it('should propagate errors from BuildVerifier', done => { + it('should propagate errors from BuildVerifier', async () => { bvGetPrIsTrustedSpy.and.callFake(() => Promise.reject('Test')); const req = createRequest(+pr).expect(500, 'Test'); - promisifyRequest(req). - then(() => { - expect(bvGetPrIsTrustedSpy).toHaveBeenCalledWith(9); - expect(bcUpdatePrVisibilitySpy).not.toHaveBeenCalled(); - }). - then(done, done.fail); + await promisifyRequest(req); + expect(bvGetPrIsTrustedSpy).toHaveBeenCalledWith(9); + expect(bcUpdatePrVisibilitySpy).not.toHaveBeenCalled(); }); - it('should call \'BuildCreator#updatePrVisibility()\' with the correct arguments', done => { + it('should call \'BuildCreator#updatePrVisibility()\' with the correct arguments', async () => { bvGetPrIsTrustedSpy.and.callFake((pr2: number) => Promise.resolve(pr2 === 42)); - const req1 = createRequest(24); - const req2 = createRequest(42); + await promisifyRequest(createRequest(24)); + expect(bcUpdatePrVisibilitySpy).toHaveBeenCalledWith(24, false); - Promise.all([ - promisifyRequest(req1).then(() => expect(bcUpdatePrVisibilitySpy).toHaveBeenCalledWith('24', false)), - promisifyRequest(req2).then(() => expect(bcUpdatePrVisibilitySpy).toHaveBeenCalledWith('42', true)), - ]).then(done, done.fail); + await promisifyRequest(createRequest(42)); + expect(bcUpdatePrVisibilitySpy).toHaveBeenCalledWith(42, true); }); - it('should propagate errors from BuildCreator', done => { + it('should propagate errors from BuildCreator', async () => { bcUpdatePrVisibilitySpy.and.callFake(() => Promise.reject('Test')); const req = createRequest(+pr).expect(500, 'Test'); - verifyRequests([req], done); + await verifyRequests([req]); }); describe('on success', () => { - it('should respond with 200 (action: undefined)', done => { + it('should respond with 200 (action: undefined)', async () => { bvGetPrIsTrustedSpy.and.returnValues(Promise.resolve(true), Promise.resolve(false)); const reqs = [4, 2].map(num => createRequest(num).expect(200, http.STATUS_CODES[200])); - verifyRequests(reqs, done); + await verifyRequests(reqs); }); - it('should respond with 200 (action: labeled)', done => { + it('should respond with 200 (action: labeled)', async () => { bvGetPrIsTrustedSpy.and.returnValues(Promise.resolve(true), Promise.resolve(false)); const reqs = [4, 2].map(num => createRequest(num, 'labeled').expect(200, http.STATUS_CODES[200])); - verifyRequests(reqs, done); + await verifyRequests(reqs); }); - it('should respond with 200 (action: unlabeled)', done => { + it('should respond with 200 (action: unlabeled)', async () => { bvGetPrIsTrustedSpy.and.returnValues(Promise.resolve(true), Promise.resolve(false)); const reqs = [4, 2].map(num => createRequest(num, 'unlabeled').expect(200, http.STATUS_CODES[200])); - verifyRequests(reqs, done); + await verifyRequests(reqs); }); - it('should respond with 200 (and do nothing) if \'action\' implies no visibility change', done => { + it('should respond with 200 (and do nothing) if \'action\' implies no visibility change', async () => { const promises = ['foo', 'notlabeled']. map(action => createRequest(+pr, action).expect(200, http.STATUS_CODES[200])). map(promisifyRequest); - Promise.all(promises). - then(() => { - expect(bvGetPrIsTrustedSpy).not.toHaveBeenCalled(); - expect(bcUpdatePrVisibilitySpy).not.toHaveBeenCalled(); - }). - then(done, done.fail); + await Promise.all(promises); + expect(bvGetPrIsTrustedSpy).not.toHaveBeenCalled(); + expect(bcUpdatePrVisibilitySpy).not.toHaveBeenCalled(); }); }); @@ -584,16 +579,16 @@ describe('uploadServerFactory', () => { describe('ALL *', () => { - it('should respond with 404', done => { + it('should respond with 404', async () => { const responseFor = (method: string) => `Unknown resource in request: ${method.toUpperCase()} /some/url`; - verifyRequests([ + await verifyRequests([ agent.get('/some/url').expect(404, responseFor('get')), agent.put('/some/url').expect(404, responseFor('put')), agent.post('/some/url').expect(404, responseFor('post')), agent.patch('/some/url').expect(404, responseFor('patch')), agent.delete('/some/url').expect(404, responseFor('delete')), - ], done); + ]); }); }); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/utils.spec.ts b/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/utils.spec.ts new file mode 100644 index 0000000000..a3fcc1e6c3 --- /dev/null +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/test/upload-server/utils.spec.ts @@ -0,0 +1,55 @@ +import * as express from 'express'; +import {UploadError} from '../../lib/upload-server/upload-error'; +import {respondWithError, throwRequestError} from '../../lib/upload-server/utils'; + +describe('upload-server/utils', () => { + describe('respondWithError', () => { + let endSpy: jasmine.Spy; + let statusSpy: jasmine.Spy; + let response: express.Response; + + beforeEach(() => { + endSpy = jasmine.createSpy('end'); + statusSpy = jasmine.createSpy('status').and.callFake(() => response); + response = {status: statusSpy, end: endSpy} as any; + }); + + it('should set the status on the response', () => { + respondWithError(response, new UploadError(505, 'TEST MESSAGE')); + + expect(statusSpy).toHaveBeenCalledWith(505); + expect(endSpy).toHaveBeenCalledWith('TEST MESSAGE', jasmine.any(Function)); + expect(console.error).toHaveBeenCalledWith('Upload error: 505 - HTTP Version Not Supported'); + expect(console.error).toHaveBeenCalledWith('TEST MESSAGE'); + }); + + it('should convert non-UploadError errors to 500 UploadErrors', () => { + respondWithError(response, new Error('OTHER MESSAGE')); + + expect(statusSpy).toHaveBeenCalledWith(500); + expect(endSpy).toHaveBeenCalledWith('OTHER MESSAGE', jasmine.any(Function)); + expect(console.error).toHaveBeenCalledWith('Upload error: 500 - Internal Server Error'); + expect(console.error).toHaveBeenCalledWith('OTHER MESSAGE'); + }); + }); + + describe('throwRequestError', () => { + it('should throw a suitable error', () => { + let caught = false; + try { + const request = { + body: 'The request body', + method: 'POST', + originalUrl: 'some.domain.com/path', + } as express.Request; + throwRequestError(505, 'ERROR MESSAGE', request); + } catch (error) { + caught = true; + expect(error).toEqual(jasmine.any(UploadError)); + expect(error.status).toEqual(505); + expect(error.message).toEqual(`ERROR MESSAGE in request: POST some.domain.com/path "The request body"`); + } + expect(caught).toEqual(true); + }); + }); +}); diff --git a/aio/aio-builds-setup/dockerbuild/scripts-js/yarn.lock b/aio/aio-builds-setup/dockerbuild/scripts-js/yarn.lock index 73ef310751..d1ecba7b2e 100644 --- a/aio/aio-builds-setup/dockerbuild/scripts-js/yarn.lock +++ b/aio/aio-builds-setup/dockerbuild/scripts-js/yarn.lock @@ -40,12 +40,6 @@ version "2.6.0" resolved "https://registry.yarnpkg.com/@types/jasmine/-/jasmine-2.6.0.tgz#997b41a27752b4850af2683bc4a8d8222c25bd02" -"@types/jsonwebtoken@^7.2.3": - version "7.2.3" - resolved "https://registry.yarnpkg.com/@types/jsonwebtoken/-/jsonwebtoken-7.2.3.tgz#483c8f39945e1e6d308dcc51fd4aeca5208d4dca" - dependencies: - "@types/node" "*" - "@types/mime@*": version "1.3.0" resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.0.tgz#d24ffac7d1006fe68517202fb2aeba3dbe48284b" @@ -54,6 +48,18 @@ version "3.0.1" resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.1.tgz#b683eb60be358304ef146f5775db4c0e3696a550" +"@types/nock@^9.1.3": + version "9.1.3" + resolved "https://registry.yarnpkg.com/@types/nock/-/nock-9.1.3.tgz#1d445679375b9e25afd449dc56585f81729454e8" + dependencies: + "@types/node" "*" + +"@types/node-fetch@^1.6.8": + version "1.6.8" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-1.6.8.tgz#a59d8c75b300ddc3ca3eef23d449d677f9486c3d" + dependencies: + "@types/node" "*" + "@types/node@*": version "7.0.31" resolved "https://registry.yarnpkg.com/@types/node/-/node-7.0.31.tgz#80ea4d175599b2a00149c29a10a4eb2dff592e86" @@ -112,6 +118,12 @@ ansi-align@^2.0.0: dependencies: string-width "^2.0.0" +ansi-green@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/ansi-green/-/ansi-green-0.1.1.tgz#8a5d9a979e458d57c40e33580b37390b8e10d0f7" + dependencies: + ansi-wrap "0.1.0" + ansi-regex@^0.2.0, ansi-regex@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-0.2.1.tgz#0d8e946967a3d8143f93e24e298525fc1b2235f9" @@ -128,6 +140,10 @@ ansi-styles@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" +ansi-wrap@0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/ansi-wrap/-/ansi-wrap-0.1.0.tgz#a82250ddb0015e9a27ca82e82ea603bbfa45efaf" + anymatch@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-1.3.0.tgz#a3e52fa39168c825ff57b0248126ce5a8ff95507" @@ -180,6 +196,10 @@ assert-plus@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-0.2.0.tgz#d74e1b87e7affc0db8aadb7021f3fe48101ab234" +assertion-error@^1.0.1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-1.1.0.tgz#e60b6b0e8f301bd97e5375215bda406c85118c0b" + async-each@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d" @@ -208,10 +228,6 @@ balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" -base64url@2.0.0, base64url@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/base64url/-/base64url-2.0.0.tgz#eac16e03ea1438eff9423d69baa36262ed1f70bb" - bcrypt-pbkdf@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz#63bc5dcb61331b92bc05fd528953c33462a06f8d" @@ -222,6 +238,13 @@ binary-extensions@^1.0.0: version "1.8.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.8.0.tgz#48ec8d16df4377eae5fa5884682480af4d95c774" +bl@^1.0.0: + version "1.2.2" + resolved "https://registry.yarnpkg.com/bl/-/bl-1.2.2.tgz#a160911717103c07410cef63ef51b397c025af9c" + dependencies: + readable-stream "^2.3.5" + safe-buffer "^5.1.1" + block-stream@*: version "0.0.9" resolved "https://registry.yarnpkg.com/block-stream/-/block-stream-0.0.9.tgz#13ebfe778a03205cfe03751481ebb4b3300c126a" @@ -276,9 +299,20 @@ braces@^1.8.2: preserve "^0.2.0" repeat-element "^1.1.2" -buffer-equal-constant-time@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" +buffer-alloc-unsafe@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/buffer-alloc-unsafe/-/buffer-alloc-unsafe-0.1.1.tgz#ffe1f67551dd055737de253337bfe853dfab1a6a" + +buffer-alloc@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/buffer-alloc/-/buffer-alloc-1.1.0.tgz#05514d33bf1656d3540c684f65b1202e90eca303" + dependencies: + buffer-alloc-unsafe "^0.1.0" + buffer-fill "^0.1.0" + +buffer-fill@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/buffer-fill/-/buffer-fill-0.1.1.tgz#76d825c4d6e50e06b7a31eb520c04d08cc235071" bytes@3.0.0: version "3.0.0" @@ -296,6 +330,17 @@ caseless@~0.12.0: version "0.12.0" resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" +chai@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/chai/-/chai-4.1.2.tgz#0f64584ba642f0f2ace2806279f4f06ca23ad73c" + dependencies: + assertion-error "^1.0.1" + check-error "^1.0.1" + deep-eql "^3.0.0" + get-func-name "^2.0.0" + pathval "^1.0.0" + type-detect "^4.0.0" + chalk@0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/chalk/-/chalk-0.5.1.tgz#663b3a648b68b55d04690d49167aa837858f2174" @@ -316,6 +361,10 @@ chalk@^1.0.0, chalk@^1.1.1, chalk@^1.1.3: strip-ansi "^3.0.0" supports-color "^2.0.0" +check-error@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/check-error/-/check-error-1.0.2.tgz#574d312edd88bb5dd8912e9286dd6c0aed4aac82" + chokidar@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-1.7.0.tgz#798e689778151c8076b4b360e5edd28cda2bb468" @@ -476,6 +525,22 @@ debug@^2.2.0: dependencies: ms "2.0.0" +debug@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" + dependencies: + ms "2.0.0" + +deep-eql@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-3.0.1.tgz#dfc9404400ad1c8fe023e7da1df1c147c4b444df" + dependencies: + type-detect "^4.0.0" + +deep-equal@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5" + deep-extend@~0.4.0: version "0.4.2" resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.4.2.tgz#48b699c27e334bf89f10892be432f6e4c7d34a7f" @@ -488,6 +553,14 @@ delegates@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" +delete-empty@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/delete-empty/-/delete-empty-2.0.0.tgz#dcf7c4f93a98445119acd57b137d13e7af78fa39" + dependencies: + log-ok "^0.1.1" + relative "^3.0.2" + rimraf "^2.6.2" + depd@1.1.1, depd@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359" @@ -520,13 +593,6 @@ ecc-jsbn@~0.1.1: dependencies: jsbn "~0.1.0" -ecdsa-sig-formatter@1.0.9: - version "1.0.9" - resolved "https://registry.yarnpkg.com/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.9.tgz#4bc926274ec3b5abb5016e7e1d60921ac262b2a1" - dependencies: - base64url "^2.0.0" - safe-buffer "^5.0.1" - ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" @@ -535,6 +601,12 @@ encodeurl@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.1.tgz#79e3d58655346909fe6f0f45a5de68103b294d20" +end-of-stream@^1.0.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.1.tgz#ed29634d19baba463b6ce6b80a37213eab71ec43" + dependencies: + once "^1.4.0" + es6-promise@^3.3.1: version "3.3.1" resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-3.3.1.tgz#a08cdde84ccdbf34d027a1451bc91d4bcd28a613" @@ -713,6 +785,10 @@ from@~0: version "0.1.7" resolved "https://registry.yarnpkg.com/from/-/from-0.1.7.tgz#83c60afc58b9c56997007ed1a768b3ab303a44fe" +fs-constants@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -754,6 +830,10 @@ gauge@~2.7.3: strip-ansi "^3.0.1" wide-align "^1.1.0" +get-func-name@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.0.tgz#ead774abee72e20409433a066366023dd6887a41" + get-stream@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" @@ -892,7 +972,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.3, inherits@^2.0.1, inherits@~2.0.0, inherits@~2.0.1: +inherits@2, inherits@2.0.3, inherits@^2.0.1, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" @@ -1044,7 +1124,7 @@ json-stable-stringify@^1.0.1: dependencies: jsonify "~0.0.0" -json-stringify-safe@~5.0.1: +json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" @@ -1052,21 +1132,6 @@ jsonify@~0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" -jsonwebtoken@^8.0.1: - version "8.0.1" - resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-8.0.1.tgz#50daef8d0a8c7de2cd06bc1013b75b04ccf3f0cf" - dependencies: - jws "^3.1.4" - lodash.includes "^4.3.0" - lodash.isboolean "^3.0.3" - lodash.isinteger "^4.0.4" - lodash.isnumber "^3.0.3" - lodash.isplainobject "^4.0.6" - lodash.isstring "^4.0.1" - lodash.once "^4.0.0" - ms "^2.0.0" - xtend "^4.0.1" - jsprim@^1.2.2: version "1.4.0" resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.0.tgz#a3b87e40298d8c380552d8cc7628a0bb95a22918" @@ -1076,23 +1141,6 @@ jsprim@^1.2.2: json-schema "0.2.3" verror "1.3.6" -jwa@^1.1.4: - version "1.1.5" - resolved "https://registry.yarnpkg.com/jwa/-/jwa-1.1.5.tgz#a0552ce0220742cd52e153774a32905c30e756e5" - dependencies: - base64url "2.0.0" - buffer-equal-constant-time "1.0.1" - ecdsa-sig-formatter "1.0.9" - safe-buffer "^5.0.1" - -jws@^3.1.4: - version "3.1.4" - resolved "https://registry.yarnpkg.com/jws/-/jws-3.1.4.tgz#f9e8b9338e8a847277d6444b1464f61880e050a2" - dependencies: - base64url "^2.0.0" - jwa "^1.1.4" - safe-buffer "^5.0.1" - kind-of@^3.0.2: version "3.2.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" @@ -1157,10 +1205,6 @@ lodash.defaults@^3.1.2: lodash.assign "^3.0.0" lodash.restparam "^3.0.0" -lodash.includes@^4.3.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f" - lodash.isarguments@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" @@ -1169,26 +1213,6 @@ lodash.isarray@^3.0.0: version "3.0.4" resolved "https://registry.yarnpkg.com/lodash.isarray/-/lodash.isarray-3.0.4.tgz#79e4eb88c36a8122af86f844aa9bcd851b5fbb55" -lodash.isboolean@^3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6" - -lodash.isinteger@^4.0.4: - version "4.0.4" - resolved "https://registry.yarnpkg.com/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz#619c0af3d03f8b04c31f5882840b77b11cd68343" - -lodash.isnumber@^3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz#3ce76810c5928d03352301ac287317f11c0b1ffc" - -lodash.isplainobject@^4.0.6: - version "4.0.6" - resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb" - -lodash.isstring@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451" - lodash.keys@^3.0.0: version "3.1.2" resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-3.1.2.tgz#4dbc0472b156be50a0b286855d1bd0b0c656098a" @@ -1197,18 +1221,25 @@ lodash.keys@^3.0.0: lodash.isarguments "^3.0.0" lodash.isarray "^3.0.0" -lodash.once@^4.0.0: - version "4.1.1" - resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac" - lodash.restparam@^3.0.0: version "3.6.1" resolved "https://registry.yarnpkg.com/lodash.restparam/-/lodash.restparam-3.6.1.tgz#936a4e309ef330a7645ed4145986c85ae5b20805" +lodash@^4.17.5: + version "4.17.5" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.5.tgz#99a92d65c0272debe8c96b6057bc8fbfa3bed511" + lodash@^4.5.1: version "4.17.4" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" +log-ok@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/log-ok/-/log-ok-0.1.1.tgz#bea3dd36acd0b8a7240d78736b5b97c65444a334" + dependencies: + ansi-green "^0.1.1" + success-symbol "^0.1.0" + lowercase-keys@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.0.tgz#4e3366b39e7f5457e35f1324bdf6f88d0bfc7306" @@ -1288,13 +1319,13 @@ minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" -"mkdirp@>=0.5 0", mkdirp@^0.5.1: +"mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1: version "0.5.1" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" dependencies: minimist "0.0.8" -ms@2.0.0, ms@^2.0.0: +ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" @@ -1306,6 +1337,24 @@ negotiator@0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9" +nock@^9.2.5: + version "9.2.5" + resolved "https://registry.yarnpkg.com/nock/-/nock-9.2.5.tgz#c131fc8d3c4723f386be0269739638be84733f2f" + dependencies: + chai "^4.1.2" + debug "^3.1.0" + deep-equal "^1.0.0" + json-stringify-safe "^5.0.1" + lodash "^4.17.5" + mkdirp "^0.5.0" + propagate "^1.0.0" + qs "^6.5.1" + semver "^5.5.0" + +node-fetch@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.1.2.tgz#ab884e8e7e57e38a944753cec706f788d1768bb5" + node-pre-gyp@^0.6.36: version "0.6.36" resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.36.tgz#db604112cb74e0d477554e9b505b17abddfab786" @@ -1394,7 +1443,7 @@ on-finished@~2.3.0: dependencies: ee-first "1.1.1" -once@^1.3.0, once@^1.3.3: +once@^1.3.0, once@^1.3.3, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" dependencies: @@ -1457,6 +1506,10 @@ path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" +pathval@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/pathval/-/pathval-1.1.0.tgz#b942e6d4bde653005ef6b71361def8727d0645e0" + pause-stream@0.0.11: version "0.0.11" resolved "https://registry.yarnpkg.com/pause-stream/-/pause-stream-0.0.11.tgz#fe5a34b0cbce12b5aa6a2b403ee2e73b602f1445" @@ -1483,6 +1536,14 @@ process-nextick-args@~1.0.6: version "1.0.7" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3" +process-nextick-args@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" + +propagate@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/propagate/-/propagate-1.0.0.tgz#00c2daeedda20e87e3782b344adba1cddd6ad709" + proxy-addr@~1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-1.1.5.tgz#71c0ee3b102de3f202f3b64f608d173fcba1a918" @@ -1508,7 +1569,7 @@ qs@6.5.0: version "6.5.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.0.tgz#8d04954d364def3efc55b5a0793e1e2c8b1e6e49" -qs@6.5.1: +qs@6.5.1, qs@^6.5.1: version "6.5.1" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" @@ -1545,6 +1606,18 @@ rc@^1.0.1, rc@^1.1.6, rc@^1.1.7: minimist "^1.2.0" strip-json-comments "~2.0.1" +readable-stream@^2.0.0, readable-stream@^2.3.5: + version "2.3.6" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.1.4: version "2.2.11" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.11.tgz#0796b31f8d7688007ff0b93a8088d34aa17c0f72" @@ -1592,6 +1665,12 @@ registry-url@^3.0.3: dependencies: rc "^1.0.1" +relative@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/relative/-/relative-3.0.2.tgz#0dcd8ec54a5d35a3c15e104503d65375b5a5367f" + dependencies: + isobject "^2.0.0" + remove-trailing-separator@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.0.2.tgz#69b062d978727ad14dc6b56ba4ab772fd8d70511" @@ -1649,6 +1728,12 @@ rimraf@2, rimraf@^2.5.1, rimraf@^2.6.1: dependencies: glob "^7.0.5" +rimraf@^2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36" + dependencies: + glob "^7.0.5" + rx@2.3.24: version "2.3.24" resolved "https://registry.yarnpkg.com/rx/-/rx-2.3.24.tgz#14f950a4217d7e35daa71bbcbe58eff68ea4b2b7" @@ -1657,6 +1742,10 @@ safe-buffer@^5.0.1: version "5.1.0" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.0.tgz#fe4c8460397f9eaaaa58e73be46273408a45e223" +safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + safe-buffer@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.0.1.tgz#d263ca54696cd8a306b5ca6551e92de57918fbe7" @@ -1671,6 +1760,10 @@ semver@^5.0.3, semver@^5.1.0, semver@^5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" +semver@^5.5.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" + send@0.15.4: version "0.15.4" resolved "https://registry.yarnpkg.com/send/-/send-0.15.4.tgz#985faa3e284b0273c793364a35c6737bd93905b9" @@ -1710,9 +1803,9 @@ setprototypeof@1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.0.3.tgz#66567e37043eeb4f04d91bd658c0cbefb55b8e04" -shelljs@^0.7.8: - version "0.7.8" - resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.7.8.tgz#decbcf874b0d1e5fb72e14b164a9683048e9acb3" +shelljs@^0.8.1: + version "0.8.1" + resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.1.tgz#729e038c413a2254c4078b95ed46e0397154a9f1" dependencies: glob "^7.0.0" interpret "^1.0.0" @@ -1787,6 +1880,12 @@ string_decoder@~1.0.0: dependencies: safe-buffer "~5.0.1" +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + dependencies: + safe-buffer "~5.1.0" + stringstream@~0.0.4: version "0.0.5" resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" @@ -1811,6 +1910,10 @@ strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" +success-symbol@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/success-symbol/-/success-symbol-0.1.0.tgz#24022e486f3bf1cdca094283b769c472d3b72897" + superagent@^3.0.0: version "3.5.2" resolved "https://registry.yarnpkg.com/superagent/-/superagent-3.5.2.tgz#3361a3971567504c351063abeaae0faa23dbf3f8" @@ -1860,6 +1963,18 @@ tar-pack@^3.4.0: tar "^2.2.1" uid-number "^0.0.6" +tar-stream@^1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-1.6.0.tgz#a50efaa7b17760b82c27b3cae4a301a8254a5715" + dependencies: + bl "^1.0.0" + buffer-alloc "^1.1.0" + end-of-stream "^1.0.0" + fs-constants "^1.0.0" + readable-stream "^2.0.0" + to-buffer "^1.1.0" + xtend "^4.0.0" + tar@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.1.tgz#8e4d2a256c0e2185c6b18ad694aec968b83cb1d1" @@ -1882,6 +1997,10 @@ timed-out@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f" +to-buffer@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/to-buffer/-/to-buffer-1.1.1.tgz#493bd48f62d7c43fcded313a03dcadb2e1213a80" + touch@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/touch/-/touch-3.1.0.tgz#fe365f5f75ec9ed4e56825e0bb76d24ab74af83b" @@ -1939,6 +2058,10 @@ tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" +type-detect@^4.0.0: + version "4.0.8" + resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" + type-is@~1.6.15: version "1.6.15" resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.15.tgz#cab10fb4909e441c82842eafe1ad646c81804410" @@ -2047,7 +2170,7 @@ xdg-basedir@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-3.0.0.tgz#496b2cc109eca8dbacfe2dc72b603c17c5870ad4" -xtend@^4.0.1: +xtend@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"