refactor: simplify bazel saucelabs targets using karma pre-test wrapper and shared saucelabs connection between tests (#34769)

* Added a /tools/saucelabs/sauce-service.sh script that manages the sauce-connect as a service which is used by the karma-saucelabs.js wrapper to start the service.
* Added /tools/saucelabs/README.md that covers the details of SauceLabs karma testing with Bazel.

PR Close #34769
This commit is contained in:
Greg Magolan
2020-01-13 14:40:21 -08:00
committed by Andrew Kushnir
parent a5a598104f
commit dff4e1e19c
13 changed files with 672 additions and 374 deletions

View File

@ -47,12 +47,15 @@ filegroup(
],
)
# To run a karma_web_test target manually, run the "./scripts/saucelabs/run-bazel-via-tunnel.sh"
# script. Note: If you are on MacOS or Windows, you need to manually start the Saucelabs tunnel
# because the script only supports the linux Saucelabs tunnel binary. We combine all tests into
# a single "karma_web_test" target because running them as separate targets in parallel can result
# in to too many browsers being acquired at the same time. This will then result in tests being
# flaky. This target runs in CI with Saucelabs and Ivy.
# To run manually:
# Setup your SAUCE_USERNAME, SAUCE_ACCESS_KEY & SAUCE_TUNNEL_IDENTIFIER.
# If on OSX, also set SAUCE_CONNECT to the path of your `sc` binary.
# environment variables and run:
# ```
# yarn bazel run //tools/saucelabs:sauce_service_setup
# yarn bazel test //:saucelabs_unit_tests --config=saucelabs --config=ivy
# ```
# See /tools/saucelabs/README.md for more info on karma Saucelabs tests under Bazel.
karma_web_test(
name = "saucelabs_unit_tests",
# Default timeout is moderate (5min). This causes the test to be terminated while
@ -60,9 +63,10 @@ karma_web_test(
# unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
# Bazel's long timeout (15min). This ensures that Karma can shut down properly.
timeout = "long",
karma = "//tools/saucelabs:karma-saucelabs",
tags = [
"local",
"manual",
"no-remote-exec",
"saucelabs",
],
deps = [
@ -70,45 +74,48 @@ karma_web_test(
],
)
karma_web_test(
# This target runs in CI with View Engine as a Saucelabs and Bazel proof-of-concept. It's a
SAUCE_TEST_SUITE_TARGETS = [
"packages/common/http/test:test_lib",
"packages/common/http/testing/test:test_lib",
"packages/common/test:test_lib",
"packages/core/test:test_lib",
"packages/forms/test:test_lib",
"packages/http/test:test_lib",
]
[
# These target runs in CI with View Engine as a Saucelabs and Bazel proof-of-concept. It's a
# subset of the legacy saucelabs tests.
name = "saucelabs_unit_tests_poc",
# Default timeout is moderate (5min). This causes the test to be terminated while
# Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
# unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
# Bazel's long timeout (15min). This ensures that Karma can shut down properly.
timeout = "long",
tags = [
"local",
"manual",
"saucelabs",
],
deps = [
# We combine all tests into a single karma_web_test target
# as running them as separate targets in parallel leads to too many
# browsers being acquired at once in SauceLabs and the tests flake out
# TODO: this is an example subset of tests below, add all remaining angular tests
"//packages/common/http/test:test_lib",
"//packages/common/http/testing/test:test_lib",
"//packages/common/test:test_lib",
"//packages/core/test:test_lib",
"//packages/forms/test:test_lib",
"//packages/http/test:test_lib",
"//packages/zone.js/test:karma_jasmine_test_ci",
# "//packages/router/test:test_lib",
# //packages/router/test:test_lib fails with:
# IE 11.0.0 (Windows 8.1.0.0) bootstrap should restore the scrolling position FAILED
# Expected undefined to equal 5000.
# at stack (eval code:2338:11)
# at buildExpectationResult (eval code:2305:5)
# at expectationResultFactory (eval code:858:11)
# at Spec.prototype.addExpectationResult (eval code:487:5)
# at addExpectationResult (eval code:802:9)
# at Anonymous function (eval code:2252:7)
# at Anonymous function (eval code:339:25)
# at step (eval code:133:17)
# at Anonymous function (eval code:114:50)
# at fulfilled (eval code:104:47)
],
karma_web_test(
name = "saucelabs_unit_tests_poc_%s" % test.replace("/", "_").replace(":", "_").replace(".", "_"),
# Default timeout is moderate (5min). This causes the test to be terminated while
# Saucelabs browsers keep running. Ultimately resulting in failing tests and browsers
# unnecessarily being acquired. Our specified Saucelabs idle timeout is 10min, so we use
# Bazel's long timeout (15min). This ensures that Karma can shut down properly.
timeout = "long",
karma = "//tools/saucelabs:karma-saucelabs",
tags = [
"exclusive",
"manual",
"no-remote-exec",
"saucelabs",
],
deps = ["//%s" % test],
)
for test in SAUCE_TEST_SUITE_TARGETS
]
# To run manually:
# Setup your SAUCE_USERNAME, SAUCE_ACCESS_KEY & SAUCE_TUNNEL_IDENTIFIER.
# If on OSX, also set SAUCE_CONNECT to the path of your `sc` binary.
# environment variables and run:
# ```
# yarn bazel run //tools/saucelabs:sauce_service_setup
# yarn bazel test //:saucelabs_unit_tests_poc_suite --config=saucelabs
# ```
# See /tools/saucelabs/README.md for more info on karma Saucelabs tests under Bazel.
test_suite(
name = "saucelabs_unit_tests_poc_suite",
tags = ["manual"],
tests = ["//:saucelabs_unit_tests_poc_%s" % test.replace("/", "_").replace(":", "_").replace(".", "_") for test in SAUCE_TEST_SUITE_TARGETS],
)