From 6d3a25d8977e272822a1955fec37fb6c483ed395 Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Fri, 6 Dec 2019 16:17:09 +0100 Subject: [PATCH] ci: run acceptance tests on saucelabs with ivy (#34277) Currently we only run Saucelabs on PRs using the legacy View Engine build. Switching that build to Ivy is not trivial and there are various options: 1. Updating the R3 switches to use POST_R3 by default. At first glance, this doesn't look easy because the current ngtsc switch logic seems to be unidirectional (only PRE_R3 to POST_R3). 2. Updating the legacy setup to run with Ivy. This sounds like the easiest solution at first.. but it turns out to be way more complicated. Packages would need to be built with ngtsc using legacy tools (i.e. first building the compiler-cli; and then building packages) and View Engine only tests would need to be determined and filtered out. Basically it will result in re-auditing all test targets. This is contradictory to the fact that we have this information in Bazel already. 3. Creating a new job that runs tests on Saucelabs with Bazel. We specify fine-grained test targets that should run. This would be a good start (e.g. acceptance tests) and also would mean that we do not continue maintaining the legacy setup.. This commit implements the third option as it allows us to move forward with the general Bazel migration. We don't want to spend too much time on our legacy setup since it will be removed anyway in the future. PR Close #34277 --- .circleci/bazel.common.rc | 4 ++ .circleci/config.yml | 43 +++++++++++++++---- BUILD.bazel | 33 +++++++++----- packages/common/http/test/BUILD.bazel | 2 +- packages/common/http/testing/test/BUILD.bazel | 2 +- packages/common/test/BUILD.bazel | 2 +- packages/core/test/BUILD.bazel | 2 +- packages/core/test/acceptance/BUILD.bazel | 2 + packages/forms/test/BUILD.bazel | 2 +- packages/http/test/BUILD.bazel | 2 +- packages/router/test/BUILD.bazel | 2 +- packages/zone.js/test/karma_test.bzl | 2 +- scripts/saucelabs/run-bazel-via-tunnel.sh | 4 +- 13 files changed, 73 insertions(+), 29 deletions(-) diff --git a/.circleci/bazel.common.rc b/.circleci/bazel.common.rc index f70a957aea..ab84c0ed80 100644 --- a/.circleci/bazel.common.rc +++ b/.circleci/bazel.common.rc @@ -13,3 +13,7 @@ test --flaky_test_attempts=2 # More details on failures build --verbose_failures=true + +# For saucelabs tests we don't want to enable flaky test attempts. Karma has its own integrated +# retry mechanism and we do not want to retry unnecessarily if Karma already tried multiple times. +test:saucelabs --flaky_test_attempts=1 diff --git a/.circleci/config.yml b/.circleci/config.yml index 36eef246d8..6b0f0557f0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -330,7 +330,7 @@ jobs: # [BUILD.bazel](https://github.com/angular/angular/blob/ef44f51d5/BUILD.bazel#L66-L92)). # # NOTE: This is currently limited to master builds only. See the `default_workflow` configuration. - test_saucelabs_bazel: + saucelabs_view_engine: executor: name: default-executor # In order to avoid the bottleneck of having a slow host machine, we acquire a better @@ -344,9 +344,9 @@ jobs: - setup_bazel_rbe - run: name: Run Bazel tests in saucelabs - # All web tests are contained within a single //:test_web_all target for Saucelabs - # as running each set of tests as a separate target will attempt to acquire too - # many browsers on Saucelabs (7 per target currently) and some tests will always + # All web tests are contained within a single //:saucelabs_unit_tests_poc target + # for Saucelabs as running each set of tests as a separate target will attempt to acquire + # too many browsers on Saucelabs (7 per target currently) and some tests will always # fail to acquire browsers. For example: # 14 02 2019 19:52:33.170:WARN [launcher]: chrome beta on SauceLabs have not captured in 180000 ms, killing. # //packages/forms/test:web_test_sauce TIMEOUT in 315.0s @@ -355,11 +355,34 @@ jobs: --tunnel-id angular-${CIRCLE_BUILD_NUM}-${CIRCLE_NODE_INDEX} \ --username $SAUCE_USERNAME \ --key $(echo $SAUCE_ACCESS_KEY | rev) \ - yarn bazel test //:test_web_all + yarn bazel test //:saucelabs_unit_tests_poc --config=saucelabs no_output_timeout: 20m - notify_webhook_on_fail: webhook_url_env_var: SLACK_DEV_INFRA_CI_FAILURES_WEBHOOK_URL + saucelabs_ivy: + executor: + name: default-executor + # In order to avoid the bottleneck of having a slow host machine, we acquire a better + # container for this job. This is necessary because we launch a lot of browsers concurrently + # and therefore the tunnel and Karma need to process a lot of file requests and tests. + resource_class: xlarge + steps: + - custom_attach_workspace + - init_environment + - setup_circleci_bazel_config + - setup_bazel_rbe + - run: + name: Run Bazel tests on Saucelabs + # Runs the //:saucelabs_tests target with Saucelabs and Ivy. + command: | + ./scripts/saucelabs/run-bazel-via-tunnel.sh \ + --tunnel-id angular-${CIRCLE_BUILD_NUM}-${CIRCLE_NODE_INDEX} \ + --username $SAUCE_USERNAME \ + --key $(echo $SAUCE_ACCESS_KEY | rev) \ + -- yarn bazel test //:saucelabs_unit_tests --config=ivy --config=saucelabs + no_output_timeout: 20m + test_aio: # Needed because the AIO tests and the PWA score test depend on Chrome being available. executor: browsers-executor @@ -819,11 +842,14 @@ workflows: - legacy-unit-tests-saucelabs: requires: - setup - - test_saucelabs_bazel: + - saucelabs_ivy: + requires: + - test_ivy_aot + - saucelabs_view_engine: # This job is currently a PoC and a subset of `legacy-unit-tests-saucelabs`. Running on # master only to avoid wasting resources. - # - # TODO: Run this job on all branches (including PRs) as soon as it is not a PoC. + # TODO: Run this job on all branches (including PRs) as soon as it is not a PoC and + # we can remove the legacy saucelabs job. <<: *only_on_master requires: - setup @@ -876,6 +902,7 @@ workflows: - test - test_ivy_aot - integration_test + - saucelabs_ivy # Only publish if `aio`/`docs` tests using the locally built Angular packages pass - test_aio_local - test_aio_local_viewengine diff --git a/BUILD.bazel b/BUILD.bazel index c031b302ef..3aacc15542 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -47,17 +47,28 @@ filegroup( ], ) -# To run a karma_web_test target locally on SauceLabs: -# 1) have SAUCE_USERNAME, SAUCE_ACCESS_KEY (and optionally a SAUCE_TUNNEL_IDENTIFIER) set in your environment -# 2) open a sauce connection with `./scripts/saucelabs/start-tunnel.sh` -# NOTE: start-tunnel.sh uses `node_modules/sauce-connect` which is current linux specific: -# "sauce-connect": "https://saucelabs.com/downloads/sc-4.5.3-linux.tar.gz". -# On OSX or Windows you'll need to use the appropriate sauce-connect binary. -# 3) run target with `yarn bazel test --config=saucelabs ` -# NOTE: --config=saucelabs is required as it makes the SAUCE_XXX environment variables available to -# the action. See /.bazelrc. +# To run a karma_web_test target manually, run the "./scripts/saucelabs/run-bazel-via-tunnel.sh" +# script. Note: If you are on MacOS or Windows, you need to manually start the Saucelabs tunnel +# because the script only supports the linux Saucelabs tunnel binary. We combine all tests into +# a single "karma_web_test" target because running them as separate targets in parallel can result +# in to too many browsers being acquired at the same time. This will then result in tests being +# flaky. This target runs in CI with Saucelabs and Ivy. karma_web_test( - name = "test_web_all", + name = "saucelabs_unit_tests", + tags = [ + "local", + "manual", + "saucelabs", + ], + deps = [ + "//packages/core/test/acceptance:acceptance_lib", + ], +) + +karma_web_test( + # This target runs in CI with View Engine as a Saucelabs and Bazel proof-of-concept. It's a + # subset of the legacy saucelabs tests. + name = "saucelabs_unit_tests_poc", tags = [ "local", "manual", @@ -65,7 +76,7 @@ karma_web_test( ], deps = [ # We combine all tests into a single karma_web_test target - # as running them as seperate targets in parallel leads to too many + # as running them as separate targets in parallel leads to too many # browsers being acquired at once in SauceLabs and the tests flake out # TODO: this is an example subset of tests below, add all remaining angular tests "//packages/common/http/test:test_lib", diff --git a/packages/common/http/test/BUILD.bazel b/packages/common/http/test/BUILD.bazel index ec114437a3..65eadfbb96 100644 --- a/packages/common/http/test/BUILD.bazel +++ b/packages/common/http/test/BUILD.bazel @@ -6,7 +6,7 @@ ts_library( srcs = glob( ["**/*.ts"], ), - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], deps = [ "//packages/common/http", diff --git a/packages/common/http/testing/test/BUILD.bazel b/packages/common/http/testing/test/BUILD.bazel index a0a42f8be2..d9de470171 100644 --- a/packages/common/http/testing/test/BUILD.bazel +++ b/packages/common/http/testing/test/BUILD.bazel @@ -6,7 +6,7 @@ ts_library( srcs = glob( ["**/*.ts"], ), - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], deps = [ "//packages/common/http", diff --git a/packages/common/test/BUILD.bazel b/packages/common/test/BUILD.bazel index f444bbf315..87e8552355 100644 --- a/packages/common/test/BUILD.bazel +++ b/packages/common/test/BUILD.bazel @@ -6,7 +6,7 @@ ts_library( srcs = glob( ["**/*.ts"], ), - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], deps = [ "//packages/common", diff --git a/packages/core/test/BUILD.bazel b/packages/core/test/BUILD.bazel index d5e8207789..ab2ad8f7fa 100644 --- a/packages/core/test/BUILD.bazel +++ b/packages/core/test/BUILD.bazel @@ -11,7 +11,7 @@ ts_library( "**/*_node_only_spec.ts", ], ), - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], deps = [ "//packages/animations", diff --git a/packages/core/test/acceptance/BUILD.bazel b/packages/core/test/acceptance/BUILD.bazel index 9d2c5d7274..cc87ecc34a 100644 --- a/packages/core/test/acceptance/BUILD.bazel +++ b/packages/core/test/acceptance/BUILD.bazel @@ -8,6 +8,8 @@ ts_library( srcs = glob( ["**/*.ts"], ), + # Visible to //:saucelabs_unit_tests + visibility = ["//:__pkg__"], deps = [ "//packages/animations", "//packages/animations/browser", diff --git a/packages/forms/test/BUILD.bazel b/packages/forms/test/BUILD.bazel index 1bf1a831e5..679a2af5c0 100644 --- a/packages/forms/test/BUILD.bazel +++ b/packages/forms/test/BUILD.bazel @@ -4,7 +4,7 @@ ts_library( name = "test_lib", testonly = True, srcs = glob(["**/*.ts"]), - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], deps = [ "//packages/common", diff --git a/packages/http/test/BUILD.bazel b/packages/http/test/BUILD.bazel index 01d68b79c2..f7d5d2bc1d 100644 --- a/packages/http/test/BUILD.bazel +++ b/packages/http/test/BUILD.bazel @@ -4,7 +4,7 @@ ts_library( name = "test_lib", testonly = True, srcs = glob(["**/*.ts"]), - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], deps = [ "//packages/common", diff --git a/packages/router/test/BUILD.bazel b/packages/router/test/BUILD.bazel index d1eab8b6cf..30e62191e2 100644 --- a/packages/router/test/BUILD.bazel +++ b/packages/router/test/BUILD.bazel @@ -4,7 +4,7 @@ ts_library( name = "test_lib", testonly = True, srcs = glob(["**/*.ts"]), - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], deps = [ "//packages/common", diff --git a/packages/zone.js/test/karma_test.bzl b/packages/zone.js/test/karma_test.bzl index 49a451f425..8a96a9e232 100644 --- a/packages/zone.js/test/karma_test.bzl +++ b/packages/zone.js/test/karma_test.bzl @@ -127,7 +127,7 @@ def karma_test(name, env_srcs, env_deps, env_entry_point, test_srcs, test_deps, ":assets/import.html", ], tags = ["zone_karma_test"], - # Visible to //:test_web_all target + # Visible to //:saucelabs_unit_tests_poc target visibility = ["//:__pkg__"], runtime_deps = [ "@npm//karma-browserstack-launcher", diff --git a/scripts/saucelabs/run-bazel-via-tunnel.sh b/scripts/saucelabs/run-bazel-via-tunnel.sh index 349a5772a9..1d359a3443 100755 --- a/scripts/saucelabs/run-bazel-via-tunnel.sh +++ b/scripts/saucelabs/run-bazel-via-tunnel.sh @@ -158,9 +158,9 @@ set +e cd $GIT_ROOT_DIR && \ # Run bazel command with saucelabs specific environment variables passed to the action # The KARMA_WEB_TEST_MODE and SAUCE_TUNNEL_IDENTIFIER environment variables provide - # envirnment variables to be read in the karma configuration file to set correct + # environment variables to be read in the karma configuration file to set correct # configurations for karma saucelabs and browser configs. - # Usage of these envirnment variables can be seen in this repo in + # Usage of these environment variables can be seen in this repo in # /karma-js.conf.js and /browser-providers.conf.js eval "$USER_COMMAND --define=KARMA_WEB_TEST_MODE=SL_REQUIRED \ --action_env=SAUCE_USERNAME=$SAUCE_USERNAME \