From 78f9a6214cfa4248f7a4468e483b0297d6136f3b Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Fri, 4 Nov 2022 14:14:37 +0200 Subject: [PATCH] Refactor ARM64 e2e test suite - Migrate the GitHub runners to Equinix c3.large.arm64 instances - Run the test suite on all supported Kubernetes versions - Add multi-tenancy smoke test Signed-off-by: Stefan Prodan --- .github/runners/README.md | 34 ++++++++++++++++---------- .github/runners/prereq.sh | 12 ++++++--- .github/runners/runner-setup.sh | 2 +- .github/workflows/e2e-arm64.yaml | 42 +++++++++++++++++++++++++++++--- 4 files changed, 68 insertions(+), 22 deletions(-) diff --git a/.github/runners/README.md b/.github/runners/README.md index a79642347e..440c686519 100644 --- a/.github/runners/README.md +++ b/.github/runners/README.md @@ -1,24 +1,32 @@ # Flux ARM64 GitHub runners -The Flux ARM64 end-to-end tests run on Equinix instances provisioned with Docker and GitHub self-hosted runners. +The Flux ARM64 end-to-end tests run on Equinix Metal instances provisioned with Docker and GitHub self-hosted runners. ## Current instances -| Runner | Instance | Region | -|---------------|---------------------|--------| -| equinix-arm-1 | flux-equinix-arm-01 | AMS1 | -| equinix-arm-2 | flux-equinix-arm-01 | AMS1 | -| equinix-arm-3 | flux-equinix-arm-01 | AMS1 | -| equinix-arm-4 | flux-equinix-arm-02 | DFW2 | -| equinix-arm-5 | flux-equinix-arm-02 | DFW2 | -| equinix-arm-6 | flux-equinix-arm-02 | DFW2 | +| Repository | Runner | Instance | Location | +|-----------------------------|------------------|------------------------|---------------| +| flux2 | equinix-arm-dc-1 | flux-equinix-arm-dc-01 | Washington DC | +| flux2 | equinix-arm-dc-2 | flux-equinix-arm-dc-01 | Washington DC | +| flux2 | equinix-arm-da-1 | flux-equinix-arm-da-01 | Dallas | +| flux2 | equinix-arm-da-2 | flux-equinix-arm-da-01 | Dallas | +| source-controller | equinix-arm-dc-1 | flux-equinix-arm-dc-01 | Washington DC | +| source-controller | equinix-arm-da-1 | flux-equinix-arm-da-01 | Dallas | +| image-automation-controller | equinix-arm-dc-1 | flux-equinix-arm-dc-01 | Washington DC | +| image-automation-controller | equinix-arm-da-1 | flux-equinix-arm-da-01 | Dallas | + +Instance spec: +- Ampere Altra Q80-30 80-core processor @ 2.8GHz +- 2 x 960GB NVME +- 256GB RAM +- 2 x 25Gbps ## Instance setup In order to add a new runner to the GitHub Actions pool, first create a server on Equinix with the following configuration: -- Type: c2.large.arm -- OS: Ubuntu 20.04 +- Type: `c3.large.arm64` +- OS: `Ubuntu 22.04 LTS` ### Install prerequisites @@ -54,14 +62,14 @@ sudo ./prereq.sh - Retrieve the GitHub runner token from the repository [settings page](https://github.com/fluxcd/flux2/settings/actions/runners/new?arch=arm64&os=linux) -- Create 3 directories `runner1`, `runner2`, `runner3` +- Create two directories `flux2-01`, `flux2-02` - In each dir run: ```shell curl -sL https://raw.githubusercontent.com/fluxcd/flux2/main/.github/runners/runner-setup.sh > runner-setup.sh \ && chmod +x ./runner-setup.sh -./runner-setup.sh equinix-arm- +./runner-setup.sh equinix-arm- ``` - Reboot the instance diff --git a/.github/runners/prereq.sh b/.github/runners/prereq.sh index 186adb8eeb..043876eba3 100755 --- a/.github/runners/prereq.sh +++ b/.github/runners/prereq.sh @@ -18,11 +18,11 @@ set -eu -KIND_VERSION=0.14.0 +KIND_VERSION=0.17.0 KUBECTL_VERSION=1.24.0 -KUSTOMIZE_VERSION=4.5.4 -HELM_VERSION=3.8.2 -GITHUB_RUNNER_VERSION=2.291.1 +KUSTOMIZE_VERSION=4.5.7 +HELM_VERSION=3.10.1 +GITHUB_RUNNER_VERSION=2.298.2 PACKAGES="apt-transport-https ca-certificates software-properties-common build-essential libssl-dev gnupg lsb-release jq pkg-config" # install prerequisites @@ -31,6 +31,10 @@ apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# fix Kubernetes DNS resolution +rm /etc/resolv.conf +cat "/run/systemd/resolve/stub-resolv.conf" | sed '/search/d' > /etc/resolv.conf + # install docker curl -fsSL https://get.docker.com -o get-docker.sh \ && chmod +x get-docker.sh diff --git a/.github/runners/runner-setup.sh b/.github/runners/runner-setup.sh index cef53faf09..b13d87e2a8 100755 --- a/.github/runners/runner-setup.sh +++ b/.github/runners/runner-setup.sh @@ -22,7 +22,7 @@ RUNNER_NAME=$1 REPOSITORY_TOKEN=$2 REPOSITORY_URL=${3:-https://github.com/fluxcd/flux2} -GITHUB_RUNNER_VERSION=2.285.1 +GITHUB_RUNNER_VERSION=2.298.2 # download runner curl -o actions-runner-linux-arm64.tar.gz -L https://github.com/actions/runner/releases/download/v${GITHUB_RUNNER_VERSION}/actions-runner-linux-arm64-${GITHUB_RUNNER_VERSION}.tar.gz \ diff --git a/.github/workflows/e2e-arm64.yaml b/.github/workflows/e2e-arm64.yaml index bafea2fd7f..4c2de99ebb 100644 --- a/.github/workflows/e2e-arm64.yaml +++ b/.github/workflows/e2e-arm64.yaml @@ -3,7 +3,7 @@ name: e2e-arm64 on: workflow_dispatch: push: - branches: [ main, update-components ] + branches: [ main, update-components, e2e-arm64* ] permissions: contents: read @@ -13,6 +13,10 @@ jobs: # Hosted on Equinix # Docs: https://github.com/fluxcd/flux2/tree/main/.github/runners runs-on: [self-hosted, Linux, ARM64, equinix] + strategy: + matrix: + # Keep this list up-to-date with https://endoflife.date/kubernetes + KUBERNETES_VERSION: [ 1.23.13, 1.24.7, 1.25.3 ] steps: - name: Checkout uses: actions/checkout@v3 @@ -23,16 +27,46 @@ jobs: - name: Prepare id: prep run: | - echo ::set-output name=CLUSTER::arm64-${GITHUB_SHA:0:7}-$(date +%s) - echo ::set-output name=CONTEXT::kind-arm64-${GITHUB_SHA:0:7}-$(date +%s) + ID=${GITHUB_SHA:0:7}-${{ matrix.KUBERNETES_VERSION }}-$(date +%s) + echo "CLUSTER=arm64-${ID}" >> $GITHUB_OUTPUT - name: Build run: | make build - name: Setup Kubernetes Kind run: | - kind create cluster --name ${{ steps.prep.outputs.CLUSTER }} --kubeconfig=/tmp/${{ steps.prep.outputs.CLUSTER }} + kind create cluster \ + --wait 5m \ + --name ${{ steps.prep.outputs.CLUSTER }} \ + --kubeconfig=/tmp/${{ steps.prep.outputs.CLUSTER }} \ + --image=kindest/node:v${{ matrix.KUBERNETES_VERSION }} - name: Run e2e tests run: TEST_KUBECONFIG=/tmp/${{ steps.prep.outputs.CLUSTER }} make e2e + - name: Run multi-tenancy tests + env: + KUBECONFIG: /tmp/${{ steps.prep.outputs.CLUSTER }} + run: | + ./bin/flux install + ./bin/flux create source git flux-system \ + --interval=15m \ + --url=https://github.com/fluxcd/flux2-multi-tenancy \ + --branch=main \ + --ignore-paths="./clusters/**/flux-system/" + ./bin/flux create kustomization flux-system \ + --interval=15m \ + --source=flux-system \ + --path=./clusters/staging + kubectl -n flux-system wait kustomization/tenants --for=condition=ready --timeout=5m + kubectl -n apps wait kustomization/dev-team --for=condition=ready --timeout=1m + kubectl -n apps wait helmrelease/podinfo --for=condition=ready --timeout=1m + - name: Debug failure + if: failure() + env: + KUBECONFIG: /tmp/${{ steps.prep.outputs.CLUSTER }} + run: | + kubectl -n flux-system get all + kubectl -n flux-system describe po + kubectl -n flux-system logs deploy/source-controller + kubectl -n flux-system logs deploy/kustomize-controller - name: Cleanup if: always() run: |