run bundle(-upgrade): configure registry pod with root certificate …

…secret (#4703)

This commit adds `--ca-secret-name` so users attempting to run
bundles hosted by a registry using a custom CA can configure
the registry Pod's `opm registry add` command with the root cert file.

internal/olm/operator/registry: rename `--secret-name` to
`--pull-secret-name` to disambiguate new flag

docs/: consolidate private/custom CA registry configuration details
into an olm-integration doc

Signed-off-by: Eric Stroczynski <ericstroczynski@gmail.com>

changelog/fragments/ca-secret-name-run-bundle.yaml

@@ -0,0 +1,5 @@
+entries:
+  - description: >
+      Added `--ca-secret-name` to `run bundle` and `run bundle-upgrade` to configure
+      the registry Pod with an in-cluster certificate Secret to use TLS with a private registry.
+    kind: addition

changelog/fragments/run-bundle-with-secret.yaml

@@ -4,7 +4,7 @@ entries:
       registry objects to a non-default service account.
     kind: addition
   - description: >
-      Added `--secret-name` to `run bundle` and `run bundle-upgrade` to configure
+      Added `--pull-secret-name` to `run bundle` and `run bundle-upgrade` to configure
       the registry Pod with an in-cluster docker config Secret
       to pull bundle images from private registries.
     kind: addition

internal/olm/operator/registry/index/registry_pod.go

@@ -72,6 +72,11 @@ type RegistryPod struct {
 	// can pull bundle images from a private registry.
 	SecretName string
 
+	// SecretName holds the name of a secret for a CA cert file containing root certificates.
+	// This file is transiently added to the registry Pod's cert pool via `opm registry add --ca-file`.
+	// The secret's key for this file must be "cert.pem".
+	CASecretName string
+
 	// pod represents a kubernetes *corev1.pod that will be created on a cluster using an index image
 	pod *corev1.Pod
 
@@ -113,11 +118,11 @@ func (rp *RegistryPod) Create(ctx context.Context, cfg *operator.Configuration,
 
 	// make catalog source the owner of registry pod object
 	if err := controllerutil.SetOwnerReference(cs, rp.pod, rp.cfg.Scheme); err != nil {
-		return nil, fmt.Errorf("set registry pod owner reference: %v", err)
+		return nil, fmt.Errorf("error setting owner reference: %w", err)
 	}
 
 	if err := rp.cfg.Client.Create(ctx, rp.pod); err != nil {
-		return nil, fmt.Errorf("create registry pod: %v", err)
+		return nil, fmt.Errorf("error creating pod: %w", err)
 	}
 
 	// get registry pod key
@@ -225,37 +230,58 @@ func (rp *RegistryPod) podForBundleRegistry() (*corev1.Pod, error) {
 	}
 
 	addImagePullSecret(rp.pod, rp.SecretName)
+	addCertSecret(rp.pod, rp.CASecretName)
 
 	return rp.pod, nil
 }
 
-// addImagePullSecret creates and mounts an image pull secret volume
-// for a docker config secret "secretName" in each container in pod.
+// addImagePullSecret creates a docker config volume for secretName
+// and a volumeMount for that secret in each container in pod.
 func addImagePullSecret(pod *corev1.Pod, secretName string) {
 	if secretName == "" {
 		return
 	}
 
-	pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
+	// Require a non-legacy docker config secret.
+	volume := makeSecretVolume(secretName, corev1.KeyToPath{Key: ".dockerconfigjson", Path: ".docker/config.json"})
+	pod.Spec.Volumes = append(pod.Spec.Volumes, volume)
+
+	addVolumeMountForSecret(pod, volume.Name, "/root")
+}
+
+// addCertSecret creates and mounts a volume containing a CA root certificate
+// to pass to `opm registry add`.
+func addCertSecret(pod *corev1.Pod, secretName string) {
+	if secretName == "" {
+		return
+	}
+
+	// Ensure the secret contains a key "cert.pem".
+	volume := makeSecretVolume(secretName, corev1.KeyToPath{Key: "cert.pem", Path: "cert.pem"})
+	pod.Spec.Volumes = append(pod.Spec.Volumes, volume)
+
+	addVolumeMountForSecret(pod, volume.Name, "/certs")
+}
+
+func makeSecretVolume(secretName string, items ...corev1.KeyToPath) corev1.Volume {
+	return corev1.Volume{
 		Name: secretName,
 		VolumeSource: corev1.VolumeSource{
 			Secret: &corev1.SecretVolumeSource{
 				SecretName:  secretName,
 				DefaultMode: newInt32(0400),
 				Optional:    newBool(false),
-				Items: []corev1.KeyToPath{
-					// Require a non-legacy docker config secret.
-					{Key: ".dockerconfigjson", Path: ".docker/config.json"},
-				},
+				Items:       items,
 			},
 		},
-	})
+	}
+}
 
+func addVolumeMountForSecret(pod *corev1.Pod, secretName, mountPath string) {
 	volumeMount := corev1.VolumeMount{
-		Name:     secretName,
-		ReadOnly: true,
-		// Mount in $HOME.
-		MountPath: "/root",
+		Name:      secretName,
+		ReadOnly:  true,
+		MountPath: mountPath,
 	}
 	for i := range pod.Spec.Containers {
 		pod.Spec.Containers[i].VolumeMounts = append(pod.Spec.Containers[i].VolumeMounts, volumeMount)
@@ -274,9 +300,9 @@ func newBool(b bool) *bool {
 	return bp
 }
 
-const containerCommand = `/bin/mkdir -p {{ dirname .DBPath }} && \
+const cmdTemplate = `/bin/mkdir -p {{ dirname .DBPath }} && \
 {{- range $i, $item := .BundleItems }}
-/bin/opm registry add -d {{ $.DBPath }} -b {{ $item.ImageTag }} --mode={{ $item.AddMode }} && \
+/bin/opm registry add -d {{ $.DBPath }} -b {{ $item.ImageTag }} --mode={{ $item.AddMode }}{{ if $.CASecretName }} --ca-file=/certs/cert.pem{{ end }} && \
 {{- end }}
 /bin/opm registry serve -d {{ .DBPath }} -p {{ .GRPCPort }}
 `
@@ -291,13 +317,13 @@ func (rp *RegistryPod) getContainerCmd() (string, error) {
 	}
 
 	// add the custom dirname template function to the
-	// template's FuncMap and parse the containerCommand
-	tmp := template.Must(template.New("cmd").Funcs(funcMap).Parse(containerCommand))
+	// template's FuncMap and parse the cmdTemplate
+	t := template.Must(template.New("cmd").Funcs(funcMap).Parse(cmdTemplate))
 
-	// execute the command by applying the parsed tmp to command
+	// execute the command by applying the parsed t to command
 	// and write command output to out
 	out := &bytes.Buffer{}
-	if err := tmp.Execute(out, rp); err != nil {
+	if err := t.Execute(out, rp); err != nil {
 		return "", fmt.Errorf("parse container command: %w", err)
 	}
 

internal/olm/operator/registry/index/registry_pod_test.go

@@ -45,10 +45,10 @@ func TestCreateRegistryPod(t *testing.T) {
 
 var _ = Describe("RegistryPod", func() {
 
-	var defaultBundleItem = BundleItem{
+	var defaultBundleItems = []BundleItem{{
 		ImageTag: "quay.io/example/example-operator-bundle:0.2.0",
 		AddMode:  SemverBundleAddMode,
-	}
+	}}
 
 	Describe("creating registry pod", func() {
 
@@ -67,7 +67,7 @@ var _ = Describe("RegistryPod", func() {
 					Namespace: "test-default",
 				}
 				rp = &RegistryPod{
-					BundleItems: []BundleItem{defaultBundleItem},
+					BundleItems: defaultBundleItems,
 					IndexImage:  testIndexImageTag,
 				}
 				By("initializing the RegistryPod")
@@ -94,29 +94,35 @@ var _ = Describe("RegistryPod", func() {
 			It("should return a valid container command for one image", func() {
 				output, err := rp.getContainerCmd()
 				Expect(err).To(BeNil())
-				Expect(output).Should(Equal(containerCommandFor(defaultDBPath, []BundleItem{defaultBundleItem})))
+				Expect(output).Should(Equal(containerCommandFor(defaultDBPath, defaultBundleItems, false)))
+			})
+
+			It("should return a container command with --ca-file", func() {
+				rp.CASecretName = "foo-secret"
+				output, err := rp.getContainerCmd()
+				Expect(err).To(BeNil())
+				Expect(output).Should(Equal(containerCommandFor(defaultDBPath, defaultBundleItems, true)))
 			})
 
 			It("should return a valid container command for three images", func() {
-				bundleItems := []BundleItem{
-					defaultBundleItem,
-					{
+				bundleItems := append(defaultBundleItems,
+					BundleItem{
 						ImageTag: "quay.io/example/example-operator-bundle:0.3.0",
 						AddMode:  ReplacesBundleAddMode,
 					},
-					{
+					BundleItem{
 						ImageTag: "quay.io/example/example-operator-bundle:1.0.1",
 						AddMode:  SemverBundleAddMode,
 					},
-				}
+				)
 				rp2 := RegistryPod{
 					DBPath:      defaultDBPath,
 					GRPCPort:    defaultGRPCPort,
 					BundleItems: bundleItems,
 				}
 				output, err := rp2.getContainerCmd()
 				Expect(err).To(BeNil())
-				Expect(output).Should(Equal(containerCommandFor(defaultDBPath, bundleItems)))
+				Expect(output).Should(Equal(containerCommandFor(defaultDBPath, bundleItems, false)))
 			})
 
 			It("check pod status should return successfully when pod check is true", func() {
@@ -188,7 +194,7 @@ var _ = Describe("RegistryPod", func() {
 
 			It("checkPodStatus should return error when pod check is false and context is done", func() {
 				rp := &RegistryPod{
-					BundleItems: []BundleItem{defaultBundleItem},
+					BundleItems: defaultBundleItems,
 					IndexImage:  testIndexImageTag,
 				}
 				Expect(rp.init(cfg)).To(Succeed())
@@ -211,10 +217,14 @@ var _ = Describe("RegistryPod", func() {
 })
 
 // containerCommandFor returns the expected container command for a db path and set of bundle items.
-func containerCommandFor(dbPath string, items []BundleItem) string {
+func containerCommandFor(dbPath string, items []BundleItem, hasCA bool) string {
+	var caFlag string
+	if hasCA {
+		caFlag = " --ca-file=/certs/cert.pem"
+	}
 	additions := &strings.Builder{}
 	for _, item := range items {
-		additions.WriteString(fmt.Sprintf("/bin/opm registry add -d %s -b %s --mode=%s && \\\n", dbPath, item.ImageTag, item.AddMode))
+		additions.WriteString(fmt.Sprintf("/bin/opm registry add -d %s -b %s --mode=%s%s && \\\n", dbPath, item.ImageTag, item.AddMode, caFlag))
 	}
 	return fmt.Sprintf("/bin/mkdir -p /database && \\\n%s/bin/opm registry serve -d /database/index.db -p 50051\n", additions.String())
 }

internal/olm/operator/registry/index_image.go

@@ -64,6 +64,7 @@ type IndexImageCatalogCreator struct {
 	BundleImage   string
 	BundleAddMode index.BundleAddMode
 	SecretName    string
+	CASecretName  string
 
 	cfg *operator.Configuration
 }
@@ -78,10 +79,14 @@ func NewIndexImageCatalogCreator(cfg *operator.Configuration) *IndexImageCatalog
 }
 
 func (c *IndexImageCatalogCreator) BindFlags(fs *pflag.FlagSet) {
-	fs.StringVar(&c.SecretName, "secret-name", "",
+	fs.StringVar(&c.SecretName, "pull-secret-name", "",
 		"Name of image pull secret (\"type: kubernetes.io/dockerconfigjson\") required "+
 			"to pull bundle images. This secret *must* be both in the namespace and an "+
 			"imagePullSecret of the service account that this command is configured to run in")
+	fs.StringVar(&c.CASecretName, "ca-secret-name", "",
+		"Name of a generic secret containing a PEM root certificate file required to pull bundle images. "+
+			"This secret *must* be in the namespace that this command is configured to run in, "+
+			"and the file *must* be encoded under the key \"cert.pem\"")
 }
 
 func (c IndexImageCatalogCreator) CreateCatalog(ctx context.Context, name string) (*v1alpha1.CatalogSource, error) {
@@ -153,7 +158,7 @@ func (c IndexImageCatalogCreator) UpdateCatalog(ctx context.Context, cs *v1alpha
 
 	if prevRegistryPodName != "" {
 		if err = c.deleteRegistryPod(ctx, prevRegistryPodName); err != nil {
-			return fmt.Errorf("error cleaning up previous registry pod: %v", err)
+			return fmt.Errorf("error cleaning up previous registry: %v", err)
 		}
 	}
 
@@ -182,16 +187,17 @@ func (c IndexImageCatalogCreator) createAnnotatedRegistry(ctx context.Context, c
 	}
 	// Initialize and create registry pod
 	registryPod := index.RegistryPod{
-		BundleItems: items,
-		IndexImage:  c.IndexImage,
-		SecretName:  c.SecretName,
+		BundleItems:  items,
+		IndexImage:   c.IndexImage,
+		SecretName:   c.SecretName,
+		CASecretName: c.CASecretName,
 	}
 	if registryPod.DBPath, err = c.getDBPath(ctx); err != nil {
 		return fmt.Errorf("get database path: %v", err)
 	}
 	pod, err := registryPod.Create(ctx, c.cfg, cs)
 	if err != nil {
-		return fmt.Errorf("error creating registry pod: %v", err)
+		return err
 	}
 
 	// JSON marshal injected bundles

website/content/en/docs/building-operators/ansible/migration.md

@@ -49,8 +49,7 @@ The easy migration path is to initialize a new project, re-recreate APIs, then c
 [quay.io](https://quay.io/)) and be logged in in your command line environment.
   - `example.com` is used as the registry Docker Hub namespace in these examples.
   Replace it with another value if using a different registry or namespace.
-  - The registry/namespace must be public, or the cluster must be provisioned with an
-  [image pull secret][k8s-image-pull-sec] if the image namespace is private.
+  - [Authentication and certificates][image-reg-config] if the registry is private or uses a custom CA.
 
 ### Creating a new project
 
@@ -297,7 +296,7 @@ kubectl logs deployment.apps/memcached-operator-controller-manager -n memcached-
 For further steps regarding the deployment of the operator, creation of custom resources, and cleaning up of resources, see the [tutorial][tutorial-deploy].
 
 [install-guide]: /docs/building-operators/ansible/installation
-[k8s-image-pull-sec]:https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+[image-reg-config]:/docs/olm-integration/cli-overview#private-bundle-and-catalog-image-registries
 [kustomize]: https://github.com/kubernetes-sigs/kustomize
 [kube-auth-proxy]: https://github.com/brancz/kube-rbac-proxy
 [metrics]: https://book.kubebuilder.io/reference/metrics.html?highlight=metr#metrics
@@ -306,4 +305,3 @@ For further steps regarding the deployment of the operator, creation of custom r
 [testing-guide]: /docs/building-operators/ansible/testing-guide
 [migration-doc]: /docs/upgrading-sdk-version/
 [tutorial-deploy]: /docs/building-operators/ansible/tutorial/#run-the-operator
-

website/content/en/docs/building-operators/ansible/quickstart.md

@@ -15,8 +15,7 @@ This guide walks through an example of building a simple memcached-operator powe
 [quay.io](https://quay.io/)) and be logged in in your command line environment.
   - `example.com` is used as the registry Docker Hub namespace in these examples.
   Replace it with another value if using a different registry or namespace.
-  - The registry/namespace must be public, or the cluster must be provisioned with an
-  [image pull secret][k8s-image-pull-sec] if the image namespace is private.
+  - [Authentication and certificates][image-reg-config] if the registry is private or uses a custom CA.
 
 
 ## Steps
@@ -56,14 +55,10 @@ This guide walks through an example of building a simple memcached-operator powe
   make bundle-build bundle-push
   ```
 
-1. Run your bundle. If your bundle image is hosted in a private registry,
-add the image pull secret for that registry host to the service account in use
-and set `--secret-name` to the secret name:
-<!-- TODO(estroz): remove the service account requirement once OLM releases a patch or new
-minor release containing https://github.com/operator-framework/operator-lifecycle-manager/pull/1941 -->
+1. Run your bundle. If your bundle image is hosted in a registry that is private and/or
+has a custom CA, these [configuration steps][image-reg-config] must be complete.
 
   ```sh
-  kubectl patch serviceaccount default -p '{"imagePullSecrets":[{"name":"<reg secret name>"}]}'
   operator-sdk run bundle example.com/memcached-operator-bundle:v0.0.1
   ```
 
@@ -110,6 +105,6 @@ Read the [full tutorial][tutorial] for an in-depth walkthough of building a Ansi
 
 [ansible-link]:https://www.ansible.com/
 [install-guide]:/docs/building-operators/ansible/installation
+[image-reg-config]:/docs/olm-integration/cli-overview#private-bundle-and-catalog-image-registries
 [doc-olm]:/docs/olm-integration/quickstart-bundle/#enabling-olm
 [tutorial]:/docs/building-operators/ansible/tutorial/
-[k8s-image-pull-sec]:https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/

website/content/en/docs/building-operators/ansible/tutorial.md

@@ -16,8 +16,7 @@ please [migrate][migration-guide], or consult the [legacy docs][legacy-quickstar
 [quay.io](https://quay.io/)) and be logged in in your command line environment.
   - `example.com` is used as the registry Docker Hub namespace in these examples.
   Replace it with another value if using a different registry or namespace.
-  - The registry/namespace must be public, or the cluster must be provisioned with an
-  [image pull secret][k8s-image-pull-sec] if the image namespace is private.
+  - [Authentication and certificates][image-reg-config] if the registry is private or uses a custom CA.
 
 
 ## Overview
@@ -209,14 +208,10 @@ in the `bundle` directory containing manifests and metadata defining your operat
 make bundle bundle-build bundle-push
 ```
 
-Finally, run your bundle. If your bundle image is hosted in a private registry,
-add the image pull secret for that registry host to the service account in use
-and set `--secret-name` to the secret name:
-<!-- TODO(estroz): remove the service account requirement once OLM releases a patch or new
-minor release containing https://github.com/operator-framework/operator-lifecycle-manager/pull/1941 -->
+Finally, run your bundle. If your bundle image is hosted in a registry that is private and/or
+has a custom CA, these [configuration steps][image-reg-config] must be complete.
 
 ```sh
-kubectl patch serviceaccount default -p '{"imagePullSecrets":[{"name":"<reg secret name>"}]}'
 operator-sdk run bundle example.com/memcached-operator-bundle:v0.0.1
 ```
 
@@ -356,11 +351,11 @@ OLM will manage creation of most if not all resources required to run your opera
 [legacy-quickstart-doc]:https://v0-19-x.sdk.operatorframework.io/docs/ansible/quickstart/
 [migration-guide]:/docs/building-operators/ansible/migration
 [install-guide]:/docs/building-operators/ansible/installation
+[image-reg-config]:/docs/olm-integration/cli-overview#private-bundle-and-catalog-image-registries
 [ansible-developer-tips]:/docs/building-operators/ansible/development-tips/
 [ansible-watches]:/docs/building-operators/ansible/reference/watches
 [custom-resources]:https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/
 [layout-doc]:/docs/building-operators/ansible/reference/scaffolding
 [doc-bundle]:https://github.com/operator-framework/operator-registry/blob/v1.16.1/docs/design/operator-bundle.md#operator-bundle
 [quickstart-bundle]:/docs/olm-integration/quickstart-bundle
 [doc-olm]:/docs/olm-integration/quickstart-bundle/#enabling-olm
-[k8s-image-pull-sec]:https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/