config.yml

# Configuration file for https://circleci.com/gh/angular/components

# Note: YAML anchors allow an object to be re-used, reducing duplication.
# The ampersand declares an alias for an object, then later the `<<: *name`
# syntax dereferences it.
# See http://blog.daemonl.com/2016/02/yaml.html
# To validate changes, use an online parser, eg.
# http://yaml-online-parser.appspot.com/

var_1: &docker_image cimg/node:16.14.0
var_2: &docker-browser-image cimg/node:16.14.0-browsers

# **Note**: When updating the beginning of the cache key, also update the cache key to match
# the new cache key prefix. This allows us to take advantage of CircleCI's fallback caching.
# Read more here: https://circleci.com/docs/2.0/caching/#restoring-cache.
var_3: &cache_key v11-ng-mat-{{ checksum ".bazelversion" }}-{{ checksum "tools/postinstall/apply-patches.js" }}-{{ checksum "WORKSPACE" }}-{{ checksum "yarn.lock" }}
# We want to invalidate the cache if the postinstall patches change. In order to apply new
# patches, a clean version of the node modules is needed. Additionally, we invalidate the cache
# if the Bazel version changes. We do this because otherwise the `bazelisk` cache folder will
# contain all previously used versions and ultimately cause the cache restoring to be slower.
var_4: &cache_fallback_key v11-ng-mat-{{ checksum ".bazelversion" }}-{{ checksum "tools/postinstall/apply-patches.js" }}-

# Settings common to each job
var_5: &job_defaults
  working_directory: ~/ng
  docker:
    - image: *docker_image

# Restores the cache that could be available for the current Yarn lock file. The cache usually
# includes the node modules and the Bazel repository cache.
var_7: &restore_cache
  restore_cache:
    keys:
      - *cache_key
      - *cache_fallback_key

# Saves the cache for the current Yarn lock file. We store the node modules and the Bazel
# repository cache in order to make subsequent builds faster.
var_8: &save_cache
  save_cache:
    key: *cache_key
    paths:
      - 'node_modules'
      - '~/.cache/bazelisk'
      - '~/bazel_repository_cache'

# Decryption token that is used to decode the GCP credentials file in ".circleci/gcp_token".
var_9: &gcp_decrypt_token 'angular'

# Job step that ensures that the node module dependencies are installed and up-to-date. We use
# Yarn with the frozen lockfile option in order to make sure that lock file and package.json are
# in sync. Unlike in Travis, we don't need to manually purge the node modules if stale because
# CircleCI automatically discards the cache if the checksum of the lock file has changed.
var_10: &yarn_install
  run:
    name: 'Installing project dependencies'
    command: yarn install --frozen-lockfile --non-interactive

# Installs all dependencies but does not enforce a frozen lockfile. Helpful when
# the "package.json" is updated as part of a CI job. e.g. when setting up snapshots.
var_11: &yarn_install_loose_lockfile
  run:
    name: 'Installing project dependencies'
    command: yarn install --non-interactive

# Sets up the Bazel config which is specific for CircleCI builds.
var_12: &setup_bazel_ci_config
  run:
    name: 'Setting up Bazel configuration for CI'
    # Note: We add the remote config flag to the user bazelrc file that is not tracked
    # by Git. This is necessary to avoid stamping builds with `.with-local-changes`.
    command: echo "import %workspace%/.circleci/bazel.rc" >> ./.bazelrc.user

# Attaches the release output which has been stored in the workspace to the current job.
# https://circleci.com/docs/2.0/workflows/#using-workspaces-to-share-data-among-jobs
var_13: &attach_release_output
  attach_workspace:
    at: dist/

# Branch filter that we can specify for jobs that should only run on publish branches. This filter
# is used to ensure that not all upstream branches will be published as Github builds
# (e.g. revert branches, feature branches)
var_14: &publish_branches_filter
  branches:
    only:
      - main
      # 6.0.x, 7.1.x, etc.
      - /\d+\.\d+\.x/

# Branch filter that is usually applied to all jobs. Since there is no way within CircleCI to
# exclude a branch for all defined jobs, we need to manually specify the filters for each job.
# In order to reduce duplication we use a YAML anchor that just always excludes the "_presubmit"
# branch. We don't want to run Circle for the temporary "_presubmit" branch which is reserved
# for the caretaker.
var_15: &ignore_presubmit_branch_filter
  branches:
    ignore:
      - '_presubmit'

# Runs a script that sets up the Bazel remote execution. This will be used by jobs that run
# Bazel primarily and should benefit from remote caching and execution.
var_16: &setup_bazel_remote_execution
  run:
    name: 'Setup bazel RBE remote execution'
    command: ./scripts/bazel/setup-remote-execution.sh

# Sets up the bazel binary globally. We don't want to access bazel through Yarn and NodeJS
# because it could mean that the Bazel child process only has access to limited memory.
var_17: &setup_bazel_binary
  run:
    name: 'Setting up global Bazel binary'
    command: ./scripts/circleci/setup_bazel_binary.sh

# Sets up the Angular snapshot builds.
var_18: &setup_snapshot_builds
  run:
    name: 'Setting up Angular snapshot builds'
    command: node ./scripts/circleci/setup-angular-snapshots.js main

# Filter which ensures that jobs only run for pull requests.
var_19: &only_on_pull_requests_filter
  branches:
    only:
      - /pull\/\d+/

# Anchor for a step that notifies Slack when preceding steps failed.
var_20: &slack_notify_on_failure
  run:
    name: 'Notifying team about job failure'
    when: on_fail
    command: yarn ci-notify-slack-failure

# Branch filter that only matches the main branch.
var_21: &only_main_branch_filter
  branches:
    only:
      - main

# -----------------------------
# Container version of CircleCI
# -----------------------------
version: 2.1

# Configures CircleCI orbs for this pipeline. Orbs allow consumption of publicly shared
# CircleCI commands, jobs, configuration elements or executors. Read more about orbs here:
# https://circleci.com/docs/2.0/orb-intro/
orbs:
  # Set up the `queue` orb that allows us to queue up builds across jobs. We use it
  # to make sure that snapshot builds are not deployed out of order, resulting in Git
  # push conflicts.
  queue: eddiewebb/queue@1.5.0

commands:
  checkout_and_rebase:
    description: Checkout and rebase the repository
    steps:
      - checkout
      # After checkout, rebase on top of target branch.
      - run:
          name: Rebase PR on target branch
          environment:
            CIRCLE_GIT_BASE_REVISION: << pipeline.git.base_revision >>
            CIRCLE_GIT_REVISION: << pipeline.git.revision >>
          command: |
            if [ -n "$CIRCLE_PR_NUMBER" ]; then
              # User is required for rebase.
              git config user.name "angular-ci"
              git config user.email "angular-ci"
              node .circleci/rebase-pr.js
            fi

# -----------------------------------------------------------------------------------------
# Job definitions. Jobs which are defined just here, will not run automatically. Each job
# must be part of a workflow definition in order to run for PRs and push builds.
# -----------------------------------------------------------------------------------------
jobs:
  # -----------------------------------
  # Job to test that everything builds with Bazel
  # -----------------------------------
  bazel_build:
    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
      - *yarn_install
      - *setup_bazel_binary

      # Exclude release and docs packages here as those will be built within
      # the "build_release_packages" and "publish_snapshots" jobs.
      - run: bazel build --build_tag_filters=-docs-package,-release-package -- src/...
      - *slack_notify_on_failure

  # --------------------------------------------------------------------------------------------
  # Job that runs API golden tests in "tools/public_api_guard".
  # This job fails whenever an API has been updated but not explicitly approved through goldens.
  # --------------------------------------------------------------------------------------------
  api_golden_checks:
    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
      - *yarn_install
      - *setup_bazel_binary

      - run: bazel test tools/public_api_guard/...
      - *slack_notify_on_failure

  # -----------------------------------------------------------------
  # Job that runs the e2e tests with Protractor and Chromium headless
  # -----------------------------------------------------------------
  e2e_tests:
    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
      - *yarn_install
      - *setup_bazel_binary

      # Run e2e tests. Note that protractor test targets are flaky sometimes, so we run them
      # with flaky test attempts. This means that Bazel will re-run a failed e2e test target
      # a second time to ensure it's a real test failure. This improves CI stability.
      - run: yarn e2e --flaky_test_attempts=2
      - *slack_notify_on_failure

  # ------------------------------------------------------------------------------------------
  # Job that runs the unit tests on Bazel-provided browsers (Chrome and Firefox headless).
  # ------------------------------------------------------------------------------------------
  tests_browsers:
    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
      - *yarn_install
      - *setup_bazel_binary

      - run: bazel test --build_tag_filters=-e2e --test_tag_filters=-e2e --build_tests_only -- src/...
      - *slack_notify_on_failure

  # ----------------------------------------------------------------------------
  # Job that runs the unit tests on Browserstack. The browsers that will be used
  # to run the unit tests on Browserstack are set in: test/browser-providers.js
  # ----------------------------------------------------------------------------
  tests_browserstack:
    <<: *job_defaults
    resource_class: xlarge
    environment:
      BROWSER_STACK_USERNAME: 'angulardevops_uNDx4q'
      BROWSER_STACK_ACCESS_KEY: 'PXeXP2qrTvRooAFG7DPW'
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *setup_bazel_ci_config
      - *yarn_install

      - run: ./scripts/circleci/run-browserstack-tests.sh
      - *slack_notify_on_failure

  # ----------------------------------------------------------------------------
  # Job that runs the unit tests on Saucelabs. The browsers that will be used
  # to run the unit tests on Saucelabs are set in: test/browser-providers.js
  # ----------------------------------------------------------------------------
  tests_saucelabs:
    <<: *job_defaults
    resource_class: xlarge
    environment:
      SAUCE_USERNAME: 'angular-components'
      SAUCE_ACCESS_KEY: '687f2bb5cacc-3bcb-5b64-8e8a-3825fc13'
      # Note: This number should not be too high because otherwise we might run into
      # a rate limit exception.
      KARMA_PARALLEL_BROWSERS: 2
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *setup_bazel_ci_config
      - *yarn_install

      - run: ./scripts/circleci/run-saucelabs-tests.sh
      - *slack_notify_on_failure

  # ----------------------------------
  # Lint job.
  # ----------------------------------
  lint:
    <<: *job_defaults
    environment:
      CI_GIT_BASE_REVISION: << pipeline.git.base_revision >>
      CI_GIT_REVISION: << pipeline.git.revision >>
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *setup_bazel_ci_config
      - *yarn_install
      - *setup_bazel_binary

      - run:
          name: Checking package externals
          command: |
            bazel build //:package_externals
            yarn check-package-externals $(bazel info bazel-bin)/package_externals.json

      - run:
          name: Checking entry-points configuration
          command: |
            bazel build //:entry_points_manifest
            yarn check-entry-point-setup $(bazel info bazel-bin)/entry_points_manifest.json

      - run: yarn ng-dev format changed --check << pipeline.git.base_revision >>
      - run: yarn ownerslint
      - run: yarn check-mdc-tests
      - run: yarn stylelint
      - run: yarn tslint
      - run: yarn -s ts-circular-deps:check
      - run: yarn check-mdc-exports
      # Should be last so that it doesn't prevent the other lint checks from running.
      - run:
          name: Check Commit Message (PR Only)
          # Only run the commit message checks on pull requests since we can't do
          # much about any failures that have made it into the main branch.
          command: |
            if [ -n "$CIRCLE_PR_NUMBER" ]; then
              yarn ng-dev commit-message validate-range $CI_GIT_BASE_REVISION $CI_GIT_REVISION
            fi

      - *slack_notify_on_failure
      - *save_cache

  # -------------------------------------------------------------------------------------------
  # Job that builds all release packages. The built packages can be then used in the same
  # workflow to publish snapshot builds.
  # -------------------------------------------------------------------------------------------
  build_release_packages:
    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
      - *yarn_install
      - *setup_bazel_binary

      - run: yarn build-and-check-release-output
      - run: yarn check-tooling-setup

      # TODO(devversion): replace this with bazel tests that run Madge. This is
      # cumbersome and doesn't guarantee no circular deps for other entry-points.
      - run: yarn madge --circular dist/releases/cdk/schematics/index.js

      # Store the release output in the workspace storage. This means that other jobs
      # in the same workflow can attach the release output to their job.
      - persist_to_workspace:
          root: dist
          paths:
            - 'releases/**/*'
      - *slack_notify_on_failure

  upload_release_packages:
    <<: *job_defaults
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *attach_release_output
      - *yarn_install

      # Creates package archives and passes in an appropriate meaningful archive suffix. The
      # suffix consists of the pull request number and a short SHA describing the current `HEAD`.
      - run: ./scripts/create-package-archives.js --suffix "pr$CIRCLE_PR_NUMBER-$(git rev-parse --short HEAD)"
      # Upload archives to the CircleCI job artifacts.
      - store_artifacts:
          path: dist/release-archives

  # ----------------------------------------
  # Job that publishes the docs site
  # ----------------------------------------
  deploy_docs_site:
    docker:
      - image: *docker-browser-image
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
      - *yarn_install
      - *setup_bazel_binary

      - run: yarn ci-push-deploy-docs-app
      - *slack_notify_on_failure

  # ----------------------------------------
  # Job that monitors the docs site, ensuring
  # the docs site is online and works as expected.
  # ----------------------------------------
  monitor_docs_site:
    docker:
      - image: *docker-browser-image
    resource_class: xlarge
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *yarn_install

      - run: yarn ci-docs-monitor-test
      - *slack_notify_on_failure

  # ----------------------------------------
  # Job that publishes the build snapshots
  # ----------------------------------------
  publish_snapshots:
    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *attach_release_output
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
      - *yarn_install
      - *setup_bazel_binary

      # CircleCI has a config setting to enforce SSH for all github connections.
      # This is not compatible with our mechanism of using a Personal Access Token
      # to publish the build snapshots. In order to fix this, we unset the global option.
      - run: git config --global --unset "url.ssh://git@github.com.insteadof"

      # The components examples package is not a release package, but we publish it
      # as part of this job to the docs-content repository. It's not contained in the
      # attached release output, so we need to build it here.
      - run: yarn build-docs-content

      # Ensures that we do not push the snapshot artifacts upstream until all previous
      # snapshot build jobs have completed. This helps avoiding conflicts when multiple
      # commits have been pushed (resulting in multiple concurrent snapshot publish jobs).
      # We increase the confidence threshold so that it queries the CircleCI API a second time
      # after a delay. We do this as the CircleCI API does not refresh immediately when a job
      # completes/starts, and this will improve stability of the queue step. See source:
      # https://github.com/eddiewebb/circleci-queue/commit/5d42add5bbcff5e8ac7fe189448a61fea98b0839.
      - queue/until_front_of_line:
          confidence: '2'

      - run: ./scripts/circleci/publish-snapshots.sh
      - *slack_notify_on_failure

  # ----------------------------------------------------------------------------
  # Job that runs the browser tests against the Angular Github snapshots
  # ----------------------------------------------------------------------------
  snapshot_tests_browsers:
    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
      - *setup_snapshot_builds
      - *yarn_install_loose_lockfile
      - *setup_bazel_binary

      - run: bazel test --build_tag_filters=-e2e --test_tag_filters=-e2e --build_tests_only -- src/...
      - *slack_notify_on_failure

  # ----------------------------------------------------------------------------
  # Job that runs all Bazel integration tests.
  # ----------------------------------------------------------------------------
  integration_tests:
    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
      - *yarn_install
      - *setup_bazel_binary

      - run: yarn integration-tests
      - run:
          name: Running size integration tests (failures are reported in Slack only).
          command: |
            # If the size integration tests fail, report the failure to a dedicated #components-ci-size-tracking Slack channel.
            yarn integration-tests:size-test || yarn ci-notify-slack-failure components-ci-size-tracking
      - *slack_notify_on_failure

  # ----------------------------------------------------------------------------
  # Job that runs the AOT linker tests.
  # ----------------------------------------------------------------------------
  linker_aot_test:
    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
      - *yarn_install
      - *setup_bazel_binary

      - run: yarn test-linker-aot
      - *slack_notify_on_failure

  # ----------------------------------------------------------------------------
  # Job that runs the JIT linker tests.
  # ----------------------------------------------------------------------------
  linker_jit_test:
    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
      - *yarn_install
      - *setup_bazel_binary

      - run: yarn test-linker-jit
      - *slack_notify_on_failure

  # ----------------------------------------------------------------------------
  # Job that runs both AOT and JIT linker tests against Angular snapshot builds.
  # ----------------------------------------------------------------------------
  snapshot_linker_tests:
    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
      - *setup_snapshot_builds
      - *yarn_install_loose_lockfile
      - *setup_bazel_binary

      - run: yarn test-linker-aot
      - run: yarn test-linker-jit
      - *slack_notify_on_failure

  # ----------------------------------------------------------------------------
  # Job that runs all Bazel tests against material-components-web@canary
  # ----------------------------------------------------------------------------
  mdc_snapshot_test_cronjob:
    <<: *job_defaults
    resource_class: xlarge
    environment:
      GCP_DECRYPT_TOKEN: *gcp_decrypt_token
    steps:
      - checkout_and_rebase
      - *restore_cache
      - *yarn_install_loose_lockfile
      - *setup_bazel_binary
      - *setup_bazel_ci_config
      - *setup_bazel_remote_execution
      - *yarn_install

      # Install the latest canary version of the "material-components-web".
      - run: node ./scripts/circleci/setup-mdc-canary.js

      # Setup the components repository to use the MDC snapshot builds.
      # Run project tests with the MDC canary builds.
      - run: bazel test --build_tag_filters=-docs-package,-e2e --test_tag_filters=-e2e --build_tests_only -- src/...
      - *slack_notify_on_failure

# ----------------------------------------------------------------------------------------
# Workflow definitions. A workflow usually groups multiple jobs together. This is useful if
# one job depends on another.
#
# NOTE: When updating this configuration section, make sure to update GitHub robot
#       config to match the new workflow jobs.
# ----------------------------------------------------------------------------------------
workflows:
  version: 2

  default_workflow:
    jobs:
      - bazel_build:
          filters: *ignore_presubmit_branch_filter
      - api_golden_checks:
          filters: *ignore_presubmit_branch_filter
      - integration_tests:
          filters: *ignore_presubmit_branch_filter
      - linker_aot_test:
          filters: *ignore_presubmit_branch_filter
      - linker_jit_test:
          filters: *ignore_presubmit_branch_filter
      - tests_browsers:
          filters: *ignore_presubmit_branch_filter
      - tests_browserstack:
          filters: *ignore_presubmit_branch_filter
      - tests_saucelabs:
          filters: *ignore_presubmit_branch_filter
      - e2e_tests:
          filters: *ignore_presubmit_branch_filter
      - build_release_packages:
          filters: *ignore_presubmit_branch_filter
      - upload_release_packages:
          # We don't want to run this job on push builds because for those, the
          # `publish_snapshots` runs, and publishes build artifacts.
          filters: *only_on_pull_requests_filter
          requires:
            - build_release_packages
      - lint:
          filters: *ignore_presubmit_branch_filter
      - publish_snapshots:
          filters: *publish_branches_filter
          requires:
            - build_release_packages
      - deploy_docs_site:
          filters: *publish_branches_filter
          requires:
            - lint
            - build_release_packages
            - tests_browsers

  # Snapshot tests workflow that is scheduled to run all specified jobs every hour.
  # This workflow runs various jobs against the Angular snapshot builds from Github.
  snapshot_tests:
    jobs:
      # Note that we need additional jobs for the cronjob snapshot tests because there
      # is no easy way to detect whether a job runs inside of a cronjob or specific
      # workflow. See: https://circleci.com/ideas/?idea=CCI-I-295
      - snapshot_tests_browsers:
          filters: *only_main_branch_filter
      - mdc_snapshot_test_cronjob:
          filters: *only_main_branch_filter
      - snapshot_linker_tests:
          filters: *only_main_branch_filter
      - monitor_docs_site:
          filters: *only_main_branch_filter

    triggers:
      - schedule:
          cron: '0 * * * *'
          filters:
            branches:
              only:
                # We only want to run the main branch against the snapshot builds because
                # it's not guaranteed that older versions of Angular Material always work
                # with the latest Angular version.
                - main