Merge pull request #8744 from leseb/fix-8734

ceph: run operator with rook user

.github/workflows/canary-integration-test.yml

@@ -68,7 +68,6 @@ jobs:
         timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr|grep -Eosq \"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\" ; do sleep 1 && echo 'waiting for the manager IP to be available'; done"
         mgr_raw=$(kubectl -n rook-ceph exec $toolbox -- ceph mgr dump -f json|jq --raw-output .active_addr)
         timeout 60 sh -c "until kubectl -n rook-ceph exec $toolbox -- curl --silent --show-error ${mgr_raw%%:*}:9283; do echo 'waiting for mgr prometheus exporter to be ready' && sleep 1; done"
-        kubectl -n rook-ceph exec $toolbox -- /bin/bash -c "echo \"$(kubectl get pods -o wide -n rook-ceph -l app=rook-ceph-mgr --no-headers=true|awk 'FNR <= 1'|awk '{print $6"\t"$1}')\" >>/etc/hosts"
         kubectl -n rook-ceph exec $toolbox -- mkdir -p /etc/ceph/test-data
         kubectl -n rook-ceph cp cluster/examples/kubernetes/ceph/test-data/ceph-status-out $toolbox:/etc/ceph/test-data/
         kubectl -n rook-ceph cp cluster/examples/kubernetes/ceph/create-external-cluster-resources.py $toolbox:/etc/ceph

PendingReleaseNotes.md

@@ -21,4 +21,4 @@ v1.8...
   (rook-ceph-mon secrets and configmap) so that the resources will not be accidentally deleted.
 - Add support for [Kubernetes Authentication when using HashiCorp Vault Key Management Service](Documentation/ceph-kms.md##kubernetes-based-authentication).
 - Bucket notification supported via CRDs
-
+- The Rook Operator and the toolbox now run under the "rook" user and does not use "root" anymore.

cluster/charts/rook-ceph/templates/deployment.yaml

@@ -26,6 +26,10 @@ spec:
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         args: ["ceph", "operator"]
+        securityContext:
+          runAsNonRoot: true
+          runAsUser: 2016
+          runAsGroup: 2016
         volumeMounts:
         - mountPath: /var/lib/rook
           name: rook-config

cluster/examples/kubernetes/ceph/operator-openshift.yaml

@@ -439,6 +439,10 @@ spec:
         - name: rook-ceph-operator
           image: rook/ceph:master
           args: ["ceph", "operator"]
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 2016
+            runAsGroup: 2016
           volumeMounts:
             - mountPath: /var/lib/rook
               name: rook-config

cluster/examples/kubernetes/ceph/operator.yaml

@@ -356,6 +356,10 @@ spec:
         - name: rook-ceph-operator
           image: rook/ceph:master
           args: ["ceph", "operator"]
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 2016
+            runAsGroup: 2016
           volumeMounts:
             - mountPath: /var/lib/rook
               name: rook-config

cluster/examples/kubernetes/ceph/toolbox.yaml

@@ -23,6 +23,10 @@ spec:
           args: ["-m", "-c", "/usr/local/bin/toolbox.sh"]
           imagePullPolicy: IfNotPresent
           tty: true
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 2016
+            runAsGroup: 2016
           env:
             - name: ROOK_CEPH_USERNAME
               valueFrom:

images/ceph/Dockerfile

@@ -20,5 +20,7 @@ COPY rook toolbox.sh set-ceph-debug-level /usr/local/bin/
 COPY ceph-monitoring /etc/ceph-monitoring
 COPY rook-external /etc/rook-external/
 COPY ceph-csv-templates /etc/ceph-csv-templates
+RUN useradd rook -u 2016 # 2016 is the UID of the rook user and also the year of the first commit in the project
+USER 2016
 ENTRYPOINT ["/usr/local/bin/rook"]
 CMD [""]

pkg/daemon/ceph/osd/kms/volumes.go

@@ -42,8 +42,10 @@ func TLSSecretVolumeAndMount(config map[string]string) []v1.VolumeProjection {
 	// Projection list
 	secretVolumeProjections := []v1.VolumeProjection{}
 
-	// File mode
-	mode := int32(0400)
+	// File mode, anybody can read, this is a must-have since the container runs as "rook" and the
+	// secret is mounted as root. There is no non-ugly way to change this behavior and it's
+	// probably as less safe as doing this mode.
+	mode := int32(0444)
 
 	// Vault TLS Secrets
 	for _, tlsOption := range cephv1.VaultTLSConnectionDetails {

pkg/daemon/ceph/osd/kms/volumes_test.go

@@ -46,7 +46,7 @@ func Test_tlsSecretPath(t *testing.T) {
 }
 
 func TestTLSSecretVolumeAndMount(t *testing.T) {
-	m := int32(0400)
+	m := int32(0444)
 	type args struct {
 		config map[string]string
 	}

pkg/operator/ceph/webhook.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 
 	"github.com/pkg/errors"
 	"github.com/rook/rook/pkg/clusterd"
@@ -59,10 +60,12 @@ func isSecretPresent(ctx context.Context, context *clusterd.Context) (bool, erro
 
 	logger.Infof("admission webhook secret %q found", admissionControllerAppName)
 	for k, data := range s.Data {
-		path := fmt.Sprintf("%s/%s", certDir, k)
-		err := ioutil.WriteFile(path, data, 0400)
+		filePath := path.Join(certDir, k)
+		// We must use 0600 mode so that the files can be overridden each time the Secret is fetched
+		// to keep an updated content
+		err := ioutil.WriteFile(filePath, data, 0600)
 		if err != nil {
-			return false, errors.Wrapf(err, "failed to write secret content to file %q", path)
+			return false, errors.Wrapf(err, "failed to write secret content to file %q", filePath)
 		}
 	}