Merge pull request #9223 from BlaineEXE/use-yq-v4

build: use yq for RBAC yaml parsing

.github/workflows/build.yml

@@ -54,17 +54,6 @@ jobs:
       working-directory: /Users/runner/go/src/github.com/rook/rook
       run: tests/scripts/validate_modified_files.sh crd
 
-    # needed for gen-rbac
-    - name: Setup python
-      uses: actions/setup-python@v2
-      with:
-        python-version: "3.9"
-
-    - name: Install pip dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install ruamel.yaml==0.17.16
-
     - name: run gen-rbac
       working-directory: /Users/runner/go/src/github.com/rook/rook
       run: GOPATH=$(go env GOPATH) make gen-rbac

.github/workflows/rbac-gen.yaml

@@ -32,16 +32,6 @@ jobs:
     - name: copy working directory to GOPATH
       run: sudo mkdir -p /home/runner/go/src/github.com && sudo cp -a /home/runner/work/rook /home/runner/go/src/github.com/
 
-    - name: Setup python
-      uses: actions/setup-python@v2
-      with:
-        python-version: "3.9"
-
-    - name: Install pip dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install ruamel.yaml==0.17.16
-
     - name: run gen-rbac
       working-directory: /home/runner/go/src/github.com/rook/rook
       run: GOPATH=$(go env GOPATH) make gen-rbac

Makefile

@@ -111,7 +111,7 @@ build.version:
 	@mkdir -p $(OUTPUT_DIR)
 	@echo "$(VERSION)" > $(OUTPUT_DIR)/version
 
-build.common: build.version helm.build mod.check
+build.common: build.version helm.build mod.check crds
 	@$(MAKE) go.init
 	@$(MAKE) go.validate
 	@$(MAKE) -C images/ceph list-image
@@ -183,7 +183,7 @@ crds: $(CONTROLLER_GEN) $(YQ)
 	@echo Updating CRD manifests
 	@build/crds/build-crds.sh $(CONTROLLER_GEN) $(YQ)
 
-gen-rbac: $(HELM) ## generate RBAC from Helm charts
+gen-rbac: $(HELM) $(YQ) ## generate RBAC from Helm charts
 	HELM=$(HELM) ./build/rbac/get-helm-rbac.sh
 
 .PHONY: all build.common cross.build.parallel

build/crds/build-crds.sh

@@ -36,6 +36,11 @@ OLM_CATALOG_DIR="${DESTINATION_ROOT}/deploy/olm/deploy/crds"
 CEPH_CRDS_FILE_PATH="${DESTINATION_ROOT}/deploy/examples/crds.yaml"
 CEPH_HELM_CRDS_FILE_PATH="${DESTINATION_ROOT}/deploy/charts/rook-ceph/templates/resources.yaml"
 
+if [[ "$($YQ_BIN_PATH --version)" != "yq (https://github.com/mikefarah/yq/) version 4."* ]]; then
+  echo "yq must be version 4.x"
+  exit 1
+fi
+
 #############
 # FUNCTIONS #
 #############
@@ -50,7 +55,7 @@ generating_crds_v1() {
   echo "Generating ceph crds"
   "$CONTROLLER_GEN_BIN_PATH" "$CRD_OPTIONS" paths="./pkg/apis/ceph.rook.io/v1" output:crd:artifacts:config="$OLM_CATALOG_DIR"
   # the csv upgrade is failing on the volumeClaimTemplate.metadata.annotations.crushDeviceClass unless we preserve the annotations as an unknown field
-  $YQ_BIN_PATH w -i "${OLM_CATALOG_DIR}"/ceph.rook.io_cephclusters.yaml spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.storage.properties.storageClassDeviceSets.items.properties.volumeClaimTemplates.items.properties.metadata.properties.annotations.x-kubernetes-preserve-unknown-fields true
+  $YQ_BIN_PATH eval --inplace '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.storage.properties.storageClassDeviceSets.items.properties.volumeClaimTemplates.items.properties.metadata.properties.annotations.x-kubernetes-preserve-unknown-fields = true' "${OLM_CATALOG_DIR}"/ceph.rook.io_cephclusters.yaml
 }
 
 generating_main_crd() {
@@ -70,8 +75,9 @@ build_helm_resources() {
     # add header
     echo "{{- if .Values.crds.enabled }}"
 
-    # Add helm annotations to all CRDS and skip the first 4 lines of crds.yaml
-    "$YQ_BIN_PATH" w -d'*' "$CEPH_CRDS_FILE_PATH" "metadata.annotations[helm.sh/resource-policy]" keep | tail -n +5
+    # Add helm annotations to all CRDS, remove empty lines in the output
+    # skip the comment lines of crds.yaml as well as the yaml doc header
+    "$YQ_BIN_PATH" eval-all '.metadata.annotations["helm.sh/resource-policy"] = "keep"' "$CEPH_CRDS_FILE_PATH" | tail -n +6
 
     # DO NOT REMOVE the empty line, it is necessary
     echo ""
@@ -91,11 +97,7 @@ fi
 
 generating_main_crd
 
-for crd in "$OLM_CATALOG_DIR/"*.yaml; do
-  echo "---" >> "$CEPH_CRDS_FILE_PATH" # yq doesn't output doc separators
-  # Process each intermediate CRD file with yq to enforce consistent formatting in the final product
-  # regardless of whether yq was used in previous steps to alter CRD intermediate files.
-  $YQ_BIN_PATH read "$crd" >> "$CEPH_CRDS_FILE_PATH"
-done
+echo "---" >> "$CEPH_CRDS_FILE_PATH" # yq doesn't output the first doc separator
+$YQ_BIN_PATH eval-all '.' "$OLM_CATALOG_DIR/"*.yaml >> "$CEPH_CRDS_FILE_PATH"
 
 build_helm_resources

build/makelib/golang.mk

@@ -199,8 +199,7 @@ $(GOJUNIT):
 	@$(GOHOST) clean -modcache
 
 export CONTROLLER_GEN=$(TOOLS_HOST_DIR)/controller-gen-$(CONTROLLER_GEN_VERSION)
-export YQ=$(TOOLS_HOST_DIR)/yq-v3
-$(CONTROLLER_GEN) $(YQ):
+$(CONTROLLER_GEN):
 	{ \
 		set -e ;\
 		mkdir -p $(TOOLS_HOST_DIR) ;\
@@ -213,12 +212,18 @@ $(CONTROLLER_GEN) $(YQ):
 		echo === installing controller-gen ;\
 		go get sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_GEN_VERSION);\
 		mv $$CONTROLLER_GEN_TMP_DIR/controller-gen $(CONTROLLER_GEN) ;\
-		echo === installing yq ;\
-		go get github.com/mikefarah/yq/v3;\
-		mv $$CONTROLLER_GEN_TMP_DIR/yq $(YQ) ;\
 		rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
 
+YQ_VERSION = v4.14.2
+YQ := $(TOOLS_HOST_DIR)/yq-$(YQ_VERSION)
+export YQ
+$(YQ):
+	@echo === installing yq $(YQ_VERSION) $(REAL_HOST_PLATFORM)
+	@mkdir -p $(TOOLS_HOST_DIR)
+	@curl -JL https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_$(REAL_HOST_PLATFORM) -o $(YQ)
+	@chmod +x $(YQ)
+
 export CODE_GENERATOR_VERSION=0.20.0
 export CODE_GENERATOR=$(TOOLS_HOST_DIR)/code-generator-$(CODE_GENERATOR_VERSION)
 $(CODE_GENERATOR):

build/rbac/get-helm-rbac.sh

@@ -8,6 +8,6 @@ pushd "$SCRIPT_DIR"
 
 ${HELM} template ../../deploy/charts/rook-ceph \
                   --namespace rook-ceph \
-                  --set crds.enabled=false | ./keep-rbac-yaml.py > rbac.yaml
+                  --set crds.enabled=false | ./keep-rbac-yaml.sh > rbac.yaml
 
 popd

build/rbac/keep-rbac-yaml.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -eEuo pipefail
+
+# READS FROM STDIN
+# WRITES TO STDOUT
+# DEBUGS TO STDERR
+
+: ${YQ:=yq}
+
+if [[ "$($YQ --version)" != "yq (https://github.com/mikefarah/yq/) version 4."* ]]; then
+  echo "yq must be version 4.x"
+  exit 1
+fi
+
+temp_dir="$(mktemp -d)"
+pushd "${temp_dir}" &>/dev/stderr
+
+# Output the RBAC into separate temporary files named with Kind and Name so that the filesystem can
+# sort the files, and we can keep the same resource ordering as before for easy diffing. Then we
+# just read in the files, sorted by the fs for final output.
+
+$YQ eval '
+    select(.kind == "PodSecurityPolicy"),
+    select(.kind == "ServiceAccount"),
+    select(.kind == "ClusterRole"),
+    select(.kind == "ClusterRoleBinding"),
+    select(.kind == "Role"),
+    select(.kind == "RoleBinding")
+  ' - | # select all RBAC resource Kinds
+$YQ eval 'del(.metadata.labels.chart)' - | # remove the 'chart' label that only applies to Helm-managed resources
+sed '/^$/d' | # remove empty lines caused by yq's display of header/footer comments
+sed '/^# Source: /d' | # helm adds '# Source: <file>' comments atop of each yaml doc. Strip these
+$YQ eval --split-exp '.kind + " " + .metadata.name + " "' - # split into files by <kind> <name> .yaml
+# outputting the filenames with spaces after kind and name keeps the same sorting from before
+
+# For debugging, output the resource kinds and names we processed and the number we are keeping
+for file in *.yml; do
+  echo "${file%.yml}" >/dev/stderr
+done
+# shellcheck disable=SC2012 # we know filenames are alphanumeric from being k8s resources
+echo "Number of RBAC resources: $(ls "${temp_dir}" | wc -l)" >/dev/stderr
+
+$YQ eval-all '.' ./*.yml | # output all files, now sorted by Kind and Name by the fs
+sed '/^$/d' # remove empty lines caused by yq's display of header/footer comments
+
+rm -rf "${temp_dir}"
+popd &>/dev/stderr