core: create rook resources with k8s recommended labels

Adding Recommended Labels on the resources created by rook and using Recommended Labels in the helm chart, for better visuals and management of k8s object Closes: rook#8400 Signed-off-by: parth-gr <paarora@redhat.com>
parth-gr · Dec 1, 2021 · de5b3d2 · de5b3d2
1 parent 4ec496d
commit de5b3d2
Show file tree

Hide file tree

Showing 27 changed files with 175 additions and 41 deletions.
diff --git a/build/rbac/keep-rbac-yaml.sh b/build/rbac/keep-rbac-yaml.sh
@@ -27,7 +27,9 @@ $YQ eval '
     select(.kind == "Role"),
     select(.kind == "RoleBinding")
   ' - | # select all RBAC resource Kinds
-$YQ eval 'del(.metadata.labels.chart)' - | # remove the 'chart' label that only applies to Helm-managed resources
+$YQ eval 'del(.metadata.labels.helm.sh/chart)' - | # remove the 'chart' label that only applies to Helm-managed resources
+$YQ eval 'del(.metadata.labels.app.kubernetes.io/managed-by)' - | # remove the 'labels.app.kubernetes.io/managed-by' label that only applies to Helm-managed resources
+$YQ eval 'del(.metadata.labels.app.kubernetes.io/created-by)' - | # remove the 'app.kubernetes.io/created-by' label that only applies to Helm-managed resources
 sed '/^$/d' | # remove empty lines caused by yq's display of header/footer comments
 sed '/^# Source: /d' | # helm adds '# Source: <file>' comments atop of each yaml doc. Strip these
 $YQ eval --split-exp '.kind + " " + .metadata.name + " "' - # split into files by <kind> <name> .yaml

diff --git a/deploy/charts/library/templates/_cluster-psp.tpl b/deploy/charts/library/templates/_cluster-psp.tpl
@@ -10,6 +10,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole

diff --git a/deploy/charts/library/templates/_cluster-serviceaccount.tpl b/deploy/charts/library/templates/_cluster-serviceaccount.tpl
@@ -12,6 +12,7 @@ metadata:
     operator: rook
     storage-backend: ceph
     i-am-a-new-label: delete-me
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 {{ include "library.imagePullSecrets" . }}
 ---
 # Service account for Ceph mgrs
@@ -23,6 +24,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 {{ include "library.imagePullSecrets" . }}
 ---
 # Service account for the job that reports the Ceph version in an image
@@ -34,6 +36,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 {{ include "library.imagePullSecrets" . }}
 ---
 # Service account for job that purges OSDs from a Rook-Ceph cluster

diff --git a/deploy/charts/library/templates/_recommended-labels.tpl b/deploy/charts/library/templates/_recommended-labels.tpl
@@ -0,0 +1,9 @@
+{{/*
+Common labels
+*/}}
+{{- define "library.rook-ceph.labels" -}}
+app.kubernetes.io/part-of: rook-ceph-operator
+app.kubernetes.io/managed-by: helm
+app.kubernetes.io/created-by: helm
+helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+{{- end -}}
diff --git a/deploy/charts/rook-ceph/templates/clusterrole.yaml b/deploy/charts/rook-ceph/templates/clusterrole.yaml
@@ -6,6 +6,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 rules:
   # Most resources are represented by a string representation of their name, such as "pods", just as it appears in the URL for the relevant API endpoint.
   # However, some Kubernetes APIs involve a "subresource", such as the logs for a pod. [...]
@@ -26,6 +27,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 rules:
 - apiGroups:
   - ""
@@ -58,6 +60,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 rules:
 - apiGroups:
   - ""
@@ -183,6 +186,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 rules:
 - apiGroups:
   - ""
@@ -239,6 +243,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 rules:
   - apiGroups: [""]
     resources: ["secrets", "configmaps"]

diff --git a/deploy/charts/rook-ceph/templates/clusterrolebinding.yaml b/deploy/charts/rook-ceph/templates/clusterrolebinding.yaml
@@ -6,6 +6,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
@@ -23,7 +24,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
-    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole

diff --git a/deploy/charts/rook-ceph/templates/deployment.yaml b/deploy/charts/rook-ceph/templates/deployment.yaml
@@ -5,7 +5,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
-    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 spec:
   replicas: 1
   selector:
@@ -15,7 +15,7 @@ spec:
     metadata:
       labels:
         app: rook-ceph-operator
-        chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+        helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
 {{- if .Values.annotations }}
       annotations:
 {{ toYaml .Values.annotations | indent 8 }}

diff --git a/deploy/charts/rook-ceph/templates/psp.yaml b/deploy/charts/rook-ceph/templates/psp.yaml
@@ -84,7 +84,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
-    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 rules:
 - apiGroups:
   - policy
@@ -102,7 +102,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
-    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole

diff --git a/deploy/charts/rook-ceph/templates/role.yaml b/deploy/charts/rook-ceph/templates/role.yaml
@@ -8,6 +8,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 rules:
 - apiGroups:
   - ""

diff --git a/deploy/charts/rook-ceph/templates/rolebinding.yaml b/deploy/charts/rook-ceph/templates/rolebinding.yaml
@@ -8,6 +8,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role

diff --git a/deploy/charts/rook-ceph/templates/serviceaccount.yaml b/deploy/charts/rook-ceph/templates/serviceaccount.yaml
@@ -7,7 +7,7 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
-    chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
+    {{- include "library.rook-ceph.labels" . | nindent 4 }}
 {{ template "library.imagePullSecrets" . }}
 ---
 # Service account for the CephFS CSI driver

diff --git a/deploy/examples/common.yaml b/deploy/examples/common.yaml
@@ -41,6 +41,10 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    app.kubernetes.io/name: rook-ceph
+    app.kubernetes.io/instance: rook-ceph
+    app.kubernetes.io/component: ceph-csi
+    app.kubernetes.io/part-of: rook-ceph-operator
 rules:
   - apiGroups:
       - ""
@@ -70,6 +74,10 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    app.kubernetes.io/name: rook-ceph
+    app.kubernetes.io/instance: rook-ceph
+    app.kubernetes.io/component: ceph-csi
+    app.kubernetes.io/part-of: rook-ceph-operator
 rules:
     # Most resources are represented by a string representation of their name, such as "pods", just as it appears in the URL for the relevant API endpoint.
     # However, some Kubernetes APIs involve a "subresource", such as the logs for a pod. [...]
@@ -91,6 +99,10 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    app.kubernetes.io/name: rook-ceph
+    app.kubernetes.io/instance: rook-ceph
+    app.kubernetes.io/component: ceph-csi
+    app.kubernetes.io/part-of: rook-ceph-operator
 rules:
   - apiGroups:
       - ""
@@ -137,6 +149,10 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    app.kubernetes.io/name: rook-ceph
+    app.kubernetes.io/instance: rook-ceph
+    app.kubernetes.io/component: ceph-csi
+    app.kubernetes.io/part-of: rook-ceph-operator
 rules:
   - apiGroups:
       - ""
@@ -262,6 +278,10 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    app.kubernetes.io/name: rook-ceph
+    app.kubernetes.io/instance: rook-ceph
+    app.kubernetes.io/component: ceph-csi
+    app.kubernetes.io/part-of: rook-ceph-operator
 rules:
   - apiGroups:
       - ""
@@ -303,6 +323,10 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    app.kubernetes.io/name: rook-ceph
+    app.kubernetes.io/instance: rook-ceph
+    app.kubernetes.io/component: ceph-csi
+    app.kubernetes.io/part-of: rook-ceph-operator
 rules:
   - apiGroups: [""]
     resources: ["secrets", "configmaps"]
@@ -359,6 +383,10 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    app.kubernetes.io/name: rook-ceph
+    app.kubernetes.io/instance: rook-ceph
+    app.kubernetes.io/component: ceph-csi
+    app.kubernetes.io/part-of: rook-ceph-operator
 # imagePullSecrets:
 # - name: my-registry-secret
 
@@ -374,6 +402,10 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    app.kubernetes.io/name: rook-ceph
+    app.kubernetes.io/instance: rook-ceph
+    app.kubernetes.io/component: ceph-csi
+    app.kubernetes.io/part-of: rook-ceph-operator
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
@@ -390,6 +422,10 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    app.kubernetes.io/name: rook-ceph
+    app.kubernetes.io/instance: rook-ceph
+    app.kubernetes.io/component: ceph-csi
+    app.kubernetes.io/part-of: rook-ceph-operator
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
@@ -407,6 +443,10 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    app.kubernetes.io/name: rook-ceph
+    app.kubernetes.io/instance: rook-ceph
+    app.kubernetes.io/component: ceph-csi
+    app.kubernetes.io/part-of: rook-ceph-operator
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole

diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml
@@ -424,6 +424,10 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    app.kubernetes.io/name: rook-ceph
+    app.kubernetes.io/instance: rook-ceph
+    app.kubernetes.io/component: rook-ceph-operator
+    app.kubernetes.io/part-of: rook-ceph-operator
 spec:
   selector:
     matchLabels:

diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml
@@ -341,6 +341,10 @@ metadata:
   labels:
     operator: rook
     storage-backend: ceph
+    app.kubernetes.io/name: rook-ceph
+    app.kubernetes.io/instance: rook-ceph
+    app.kubernetes.io/component: rook-ceph-operator
+    app.kubernetes.io/part-of: rook-ceph-operator
 spec:
   selector:
     matchLabels:

diff --git a/design/ceph/interacting-with-rook-resources.md b/design/ceph/interacting-with-rook-resources.md
@@ -0,0 +1,43 @@
+# Labels added to Rook-Ceph resources
+
+[Recommended Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/) are a common set of labels that allows tools to work interoperably, describing objects in a common manner that all tools can understand.
+
+## Labels added to all Resources created by Rook
+
+* `app.kubernetes.io/name`: Is the name of the binary running in a container(combination of "ceph-"+daemonType).
+
+* `app.kubernetes.io/instance`: A unique name identifying the instance of an application. Due to the nature of how resources are named in Rook, this is guaranteed to be unique per CephCluster namespace but not unique within the entire Kubernetes cluster.
+
+* `app.kubernetes.io/component`: This is populated with the Kind of the resource controlling this application. For example, `cephclusters.ceph.rook.io` or `cephfilesystems.ceph.rook.io`.
+
+* `app.kubernetes.io/part-of`: This is populated with the Name of the resource controlling this application.
+
+* `app.kubernetes.io/managed-by`: `rook-ceph-operator` is the tool being used to manage the operation of an application
+
+* `app.kubernetes.io/created-by`: `rook-ceph-operator` is the controller/user who created this resource
+
+* `rook.io/operator-namespace`: The namespace in which rook-ceph operator is running.
+
+An Example of Recommended Labels on Ceph mon with ID=a will look like:
+```
+	app.kubernetes.io/name       : "ceph-mon"
+	app.kubernetes.io/instance   : "a"
+	app.kubernetes.io/component  : "cephclusters.ceph.rook.io"
+	app.kubernetes.io/part-of    : "rook-ceph"
+	app.kubernetes.io/managed-by : "rook-ceph-operator"
+	app.kubernetes.io/created-by : "rook-ceph-operator"
+	rook.io/operator-namespace   : "rook-ceph"
+```
+
+Another example on CephFilesystem with ID=a:
+```
+	app.kubernetes.io/name       : "ceph-mds"
+	app.kubernetes.io/instance   : "myfs-a"
+	app.kubernetes.io/component  : "cephfilesystems.ceph.rook.io"
+	app.kubernetes.io/part-of    : "myfs"
+	app.kubernetes.io/managed-by : "rook-ceph-operator"
+	app.kubernetes.io/created-by : "rook-ceph-operator"
+	rook.io/operator-namespace   : "rook-ceph"
+``` 
+
+**NOTE** : A totally unique string for an application can be built up from (a) app.kubernetes.io/component, (b) app.kubernetes.io/part-of, (c) the resource's namespace, (d) app.kubernetes.io/name, and (e) app.kubernetes.io/instance fields. For the example above, we could join those fields with underscore connectors like this: cephclusters.ceph.rook.io_rook-ceph_rook-ceph_ceph-mon_a. Note that this full spec can easily exceed the 64-character limit imposed on Kubernetes labels.
diff --git a/pkg/operator/ceph/cluster/mgr/spec.go b/pkg/operator/ceph/cluster/mgr/spec.go
@@ -51,7 +51,7 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
 	podSpec := v1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:   mgrConfig.ResourceName,
-			Labels: c.getPodLabels(mgrConfig.DaemonID, true),
+			Labels: c.getPodLabels(mgrConfig, true),
 		},
 		Spec: v1.PodSpec{
 			InitContainers: []v1.Container{
@@ -111,11 +111,11 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      mgrConfig.ResourceName,
 			Namespace: c.clusterInfo.Namespace,
-			Labels:    c.getPodLabels(mgrConfig.DaemonID, true),
+			Labels:    c.getPodLabels(mgrConfig, true),
 		},
 		Spec: apps.DeploymentSpec{
 			Selector: &metav1.LabelSelector{
-				MatchLabels: c.getPodLabels(mgrConfig.DaemonID, false),
+				MatchLabels: c.getPodLabels(mgrConfig, false),
 			},
 			Template: podSpec,
 			Replicas: &replicas,
@@ -318,10 +318,10 @@ func (c *Cluster) makeDashboardService(name, activeDaemon string) (*v1.Service,
 	return svc, nil
 }
 
-func (c *Cluster) getPodLabels(daemonName string, includeNewLabels bool) map[string]string {
-	labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, "mgr", daemonName, includeNewLabels)
+func (c *Cluster) getPodLabels(mgrConfig *mgrConfig, includeNewLabels bool) map[string]string {
+	labels := controller.CephDaemonAppLabels(AppName, c.clusterInfo.Namespace, config.MgrType, mgrConfig.DaemonID, c.clusterInfo.NamespacedName().Name, "cephclusters.ceph.rook.io", includeNewLabels)
 	// leave "instance" key for legacy usage
-	labels["instance"] = daemonName
+	labels["instance"] = mgrConfig.DaemonID
 	return labels
 }
 

diff --git a/pkg/operator/ceph/cluster/mon/spec.go b/pkg/operator/ceph/cluster/mon/spec.go
@@ -43,7 +43,7 @@ const (
 func (c *Cluster) getLabels(monConfig *monConfig, canary, includeNewLabels bool) map[string]string {
 	// Mons have a service for each mon, so the additional pod data is relevant for its services
 	// Use pod labels to keep "mon: id" for legacy
-	labels := controller.CephDaemonAppLabels(AppName, c.Namespace, "mon", monConfig.DaemonName, includeNewLabels)
+	labels := controller.CephDaemonAppLabels(AppName, c.Namespace, config.MonType, monConfig.DaemonName, c.ClusterInfo.NamespacedName().Name, "cephclusters.ceph.rook.io", includeNewLabels)
 	// Add "mon_cluster: <namespace>" for legacy
 	labels[monClusterAttr] = c.Namespace
 	if canary {