ci: refactoring from adding material tests to master ci (#28496)

This cherry-picks changes from #28197 to minimize the difference in
the ci infrastructure between master the the current patch branch

PR Close #28496

.circleci/config.yml

@@ -298,14 +298,16 @@ jobs:
           name: Wait for preview and run tests
           command: node aio/scripts/test-preview.js $CI_PULL_REQUEST $CI_COMMIT $CI_AIO_MIN_PWA_SCORE
 
-  # This job exists only for backwards-compatibility with old scripts and tests
+
-  # that rely on the pre-Bazel dist/packages-dist layout.
+  # The `build-npm-packages` tasks exist for backwards-compatibility with old scripts and
-  # It duplicates some work with the job above: we build the bazel packages
+  # tests that rely on the pre-Bazel `dist/packages-dist` output structure (build.sh).
-  # twice. Even though we have a remote cache, these jobs will typically run in
+  # Having multiple jobs that independently build in this manner duplicates some work; we build
-  # parallel so up-to-date outputs will not be available at the time the build
+  # the bazel packages more than once. Even though we have a remote cache, these jobs will
+  # typically run in parallel so up-to-date outputs will not be available at the time the build
   # starts.
-  # No new jobs should depend on this one.
+
-  build-packages-dist:
+  # Build the view engine npm packages. No new jobs should depend on this.
+  build-npm-packages:
     <<: *job_defaults
     resource_class: xlarge
     steps:
@@ -326,6 +328,29 @@ jobs:
           root: dist
           paths:
             - packages-dist
+
+
+  # Build the ivy npm packages.
+  build-ivy-npm-packages:
+    <<: *job_defaults
+    resource_class: xlarge
+    steps:
+      - checkout:
+          <<: *post_checkout
+      - restore_cache:
+          key: *cache_key
+      - *define_env_vars
+      - *setup_circleci_bazel_config
+      - *yarn_install
+      - *setup_bazel_remote_execution
+
+      - run: scripts/build-ivy-npm-packages.sh
+
+      # Save the npm packages from //packages/... for other workflow jobs to read
+      # https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
+      - persist_to_workspace:
+          root: dist
+          paths:
             - packages-dist-ivy-aot
 
   # We run the integration tests outside of Bazel for now.
@@ -501,7 +526,7 @@ workflows:
       - lint
       - test
       - test_ivy_aot
-      - build-packages-dist
+      - build-npm-packages
       - test_aio
       - legacy-unit-tests-local
       - legacy-unit-tests-saucelabs
@@ -510,22 +535,19 @@ workflows:
             - test_aio
       - legacy-e2e-tests:
           requires:
-            - build-packages-dist
+            - build-npm-packages
       - legacy-misc-tests:
           requires:
-            - build-packages-dist
+            - build-npm-packages
       - test_aio_local:
           requires:
-            - build-packages-dist
+            - build-npm-packages
-      # - test_aio_local_ivy:
-      #     requires:
-      #       - build-packages-dist
       - test_aio_tools:
           requires:
-            - build-packages-dist
+            - build-npm-packages
       - test_docs_examples:
           requires:
-            - build-packages-dist
+            - build-npm-packages
       - aio_preview:
           # Only run on PR builds. (There can be no previews for non-PR builds.)
           filters:
@@ -536,7 +558,7 @@ workflows:
             - aio_preview
       - integration_test:
           requires:
-            - build-packages-dist
+            - build-npm-packages
       - publish_snapshot:
           # Note: no filters on this job because we want it to run for all upstream branches
           # We'd really like to filter out pull requests here, but not yet available:
@@ -551,9 +573,9 @@ workflows:
             - test_aio_local
             # - test_aio_local_ivy
             - test_docs_examples
-            # Get the artifacts to publish from the build-packages-dist job
+            # Get the artifacts to publish from the build-npm-packages job
             # since the publishing script expects the legacy outputs layout.
-            - build-packages-dist
+            - build-npm-packages
             - legacy-e2e-tests
             - legacy-misc-tests
             - legacy-unit-tests-local

scripts/build-ivy-npm-packages.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+source ./scripts/package-builder.sh
+
+# Build the ivy packages
+buildTargetPackages "dist/packages-dist-ivy-aot" "aot" "Ivy AOT"

scripts/build-packages-dist.sh

@@ -1,69 +1,6 @@
 #!/usr/bin/env bash
-# Build the dist/packages-dist directory in the same fashion as the legacy
-# /build.sh script, by building the npm packages with Bazel and copying files.
-# This is needed for scripts and tests which are not updated to the Bazel output
-# layout (which always matches the input layout).
-# Do not add new dependencies on this script, instead adapt scripts to use the
-# new layout, and write new tests as Bazel targets.
 
-set -u -e -o pipefail
+source ./scripts/package-builder.sh
 
-cd "$(dirname "$0")"
+# Build the legacy (view engine) npm packages into dist/packages-dist
-
+buildTargetPackages "dist/packages-dist" "legacy" "Production"
-# basedir is the workspace root
-readonly basedir=$(pwd)/..
-# We need to resolve the Bazel binary in the node modules because running Bazel
-# through `yarn bazel` causes additional output that throws off command stdout.
-readonly bazelBin=$(yarn bin)/bazel
-readonly bin=$(${bazelBin} info bazel-bin)
-
-function buildTargetPackages() {
-  targets="$1"
-  destPath="$2"
-  compileMode="$3"
-  desc="$4"
-
-  echo "##################################"
-  echo "scripts/build-packages-dist.sh:"
-  echo "  building @angular/* npm packages"
-  echo "  mode: ${desc}"
-  echo "##################################"
-
-  # Use --config=release so that snapshot builds get published with embedded version info
-  echo "$targets" | xargs ${bazelBin} build --config=release --define=compile=$compileMode
-
-  [ -d "${basedir}/${destPath}" ] || mkdir -p $basedir/${destPath}
-
-  dirs=`echo "$targets" | sed -e 's/\/\/packages\/\(.*\):npm_package/\1/'`
-
-  for pkg in $dirs; do
-    # Skip any that don't have an "npm_package" target
-    srcDir="${bin}/packages/${pkg}/npm_package"
-    destDir="${basedir}/${destPath}/${pkg}"
-    if [ -d $srcDir ]; then
-      echo "# Copy artifacts to ${destDir}"
-      rm -rf $destDir
-      cp -R $srcDir $destDir
-      chmod -R u+w $destDir
-    fi
-  done
-}
-
-# Ideally these integration tests should run under bazel, and just list the npm
-# packages in their deps[].
-# Until then, we have to manually run bazel first to create the npm packages we
-# want to test.
-BAZEL_TARGETS=`${bazelBin} query --output=label 'attr("tags", "\[.*release-with-framework.*\]", //packages/...) intersect kind(".*_package", //packages/...)'`
-buildTargetPackages "$BAZEL_TARGETS" "dist/packages-dist" "legacy" "Production"
-
-# We don't use the ivy build in the integration tests, only when publishing
-# snapshots.
-# This logic matches what we use in the .circleci/config.yml file to short-
-# circuit execution of the publish-packages job.
-[[  "${CI_PULL_REQUEST-}" != "false"
-    || "${CI_REPO_OWNER-}" != "angular"
-    || "${CI_REPO_NAME-}" != "angular"
-    || "${CI_BRANCH}" != "master"
-]] && exit 0
-
-buildTargetPackages "$BAZEL_TARGETS" "dist/packages-dist-ivy-aot" "aot" "Ivy AOT"

scripts/package-builder.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+# Build the dist/packages-dist directory in the same fashion as the legacy
+# /build.sh script, by building the npm packages with Bazel and copying files.
+# This is needed for scripts and tests which are not updated to the Bazel output
+# layout (which always matches the input layout).
+# Do not add new dependencies on this script, instead adapt scripts to use the
+# new layout, and write new tests as Bazel targets.
+#
+# Ideally integration tests should run under bazel, and just consume the npm
+# packages via `deps`. Until that works, we manually build the npm packages and then
+# copy the results to the appropriate `dist` location.
+
+set -u -e -o pipefail
+
+cd "$(dirname "$0")"
+
+# basedir is the workspace root
+readonly base_dir=$(pwd)/..
+# We need to resolve the Bazel binary in the node modules because running Bazel
+# through `yarn bazel` causes additional output that throws off command stdout.
+readonly bazel_bin=$(yarn bin)/bazel
+readonly bin=$(${bazel_bin} info bazel-bin)
+
+function buildTargetPackages() {
+  # List of targets to build, e.g. core, common, compiler, etc.
+  targets=$(${bazel_bin} query --output=label 'attr("tags", "\[.*release-with-framework.*\]", //packages/...) intersect kind(".*_package", //packages/...)')
+
+  # Path to the output directory into which we copy the npm packages.
+  dest_path="$1"
+
+  # Either "legacy" (view engine) or "aot" (ivy)
+  compile_mode="$2"
+
+  # Human-readable description of the build.
+  desc="$3"
+
+  echo "##################################"
+  echo "scripts/build-packages-dist.sh:"
+  echo "  building @angular/* npm packages"
+  echo "  mode: ${desc}"
+  echo "##################################"
+
+  # Use --config=release so that snapshot builds get published with embedded version info
+  echo "$targets" | xargs ${bazel_bin} build --config=release --define=compile=${compile_mode}
+
+  [[ -d "${base_dir}/${dest_path}" ]] || mkdir -p ${base_dir}/${dest_path}
+
+  dirs=`echo "$targets" | sed -e 's/\/\/packages\/\(.*\):npm_package/\1/'`
+
+  for pkg in ${dirs}; do
+    # Skip any that don't have an "npm_package" target
+    src_dir="${bin}/packages/${pkg}/npm_package"
+    dest_dir="${base_dir}/${dest_path}/${pkg}"
+    if [[ -d ${src_dir} ]]; then
+      echo "# Copy artifacts to ${dest_dir}"
+      rm -rf ${dest_dir}
+      cp -R ${src_dir} ${dest_dir}
+      chmod -R u+w ${dest_dir}
+    fi
+  done
+}
+