Remove dependency of the API onto the typed cluster-api client (#2914)

* Use downstream triple package

Upstream removed the triple package[0], in order to be able to upgrade
client-go we need to have a downsream version.

[0] kubernetes/kubernetes#70966

* WIP

* Update fixtures

* Override cluster-api dependency

* Fix linting

* Split Kubermatic alerts into master and seed (#2909)

* split kubermatic alerts into master and seed

* rebuild outdated alerting rules

* Remove clusterv1alpha1 clientset dependecy from cluster controller

* Remove unused script

* Use ctrlruntimeClient in pkg/handler/node_v1.go

* Finish moving handler to ctrlruntimeClient

* Remove GetAdminKubernetesClientForCustomerCluster from Clusterprovider

* Use dynamic client for machines in conformance tester

* Fix handler tests

* Remove unused script

* Fix linting

* Actually fix linting error

* Register schemes

* Register clusterapi scheme in usercluster connection provider

* Remove redundant options to the dynamic client

* Filter list calls

* Put impersonated users into system:authenticated group

This is required for some very basic things like API discovery to work

* Dep ensure

* Add permissions to get nodes

* Rbac generator script: Use a different metrics serving port than cluster controller

* Fixup rbac for usercluster nodes

* Dont fatalf outside of main

* Print reconciling info on level four

api/cmd/conformance-tests/cleanup.go

@@ -53,7 +53,7 @@ func deleteAllNonDefaultNamespaces(log *logrus.Entry, kubeClient kubernetes.Inte
 	})
 }
 
-func tryToDeleteClusterWithRetries(log *logrus.Entry, cluster *kubermaticv1.Cluster, clusterClientProvider *clusterclient.Provider, kubermaticClient kubermaticclientset.Interface) error {
+func tryToDeleteClusterWithRetries(log *logrus.Entry, cluster *kubermaticv1.Cluster, clusterClientProvider clusterclient.UserClusterConnectionProvider, kubermaticClient kubermaticclientset.Interface) error {
 	const maxAttempts = 5
 	return retryNAttempts(maxAttempts, func(attempt int) error {
 		err := tryToDeleteCluster(log, cluster, clusterClientProvider, kubermaticClient)
@@ -66,7 +66,7 @@ func tryToDeleteClusterWithRetries(log *logrus.Entry, cluster *kubermaticv1.Clus
 
 // tryToDeleteCluster will try to delete all potential orphaned cloud provider resources like LB's & PVC's
 // After deleting them it will delete the kubermatic cluster object
-func tryToDeleteCluster(log *logrus.Entry, cluster *kubermaticv1.Cluster, clusterClientProvider *clusterclient.Provider, kubermaticClient kubermaticclientset.Interface) error {
+func tryToDeleteCluster(log *logrus.Entry, cluster *kubermaticv1.Cluster, clusterClientProvider clusterclient.UserClusterConnectionProvider, kubermaticClient kubermaticclientset.Interface) error {
 	log.Infof("Trying to delete cluster...")
 
 	kubeClient, err := clusterClientProvider.GetClient(cluster)

api/cmd/conformance-tests/main.go

@@ -59,7 +59,7 @@ type Opts struct {
 	clusterLister                  kubermaticv1lister.ClusterLister
 	kubermaticClient               kubermaticclientset.Interface
 	seedKubeClient                 kubernetes.Interface
-	clusterClientProvider          *clusterclient.Provider
+	clusterClientProvider          clusterclient.UserClusterConnectionProvider
 	dcFile                         string
 	repoRoot                       string
 	dcs                            map[string]provider.DatacenterMeta
@@ -283,7 +283,10 @@ func main() {
 
 	opts.clusterLister = kubermaticInformerFactory.Kubermatic().V1().Clusters().Lister()
 
-	clusterClientProvider := clusterclient.NewExternal(kubeInformerFactory.Core().V1().Secrets().Lister())
+	clusterClientProvider, err := clusterclient.NewExternal(kubeInformerFactory.Core().V1().Secrets().Lister())
+	if err != nil {
+		log.Fatalf("failed to get clusterClientProvider: %v", err)
+	}
 	opts.clusterClientProvider = clusterClientProvider
 
 	kubermaticInformerFactory.Start(rootCtx.Done())
@@ -307,7 +310,7 @@ func main() {
 	log.Infof("Whole suite took: %.2f seconds", time.Since(start).Seconds())
 }
 
-func cleanupClusters(opts Opts, log *logrus.Entry, kubermaticClient kubermaticclientset.Interface, clusterClientProvider *clusterclient.Provider) error {
+func cleanupClusters(opts Opts, log *logrus.Entry, kubermaticClient kubermaticclientset.Interface, clusterClientProvider clusterclient.UserClusterConnectionProvider) error {
 	if opts.namePrefix == "" {
 		log.Fatalf("cleanup-on-start was specified but name-prefix is empty")
 	}

api/cmd/conformance-tests/runner.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"encoding/xml"
 	"errors"
 	"fmt"
@@ -97,7 +98,7 @@ type testRunner struct {
 	kubermaticClient      kubermaticclientset.Interface
 	seedKubeClient        kubernetes.Interface
 	clusterLister         kubermaticv1lister.ClusterLister
-	clusterClientProvider *clusterclient.Provider
+	clusterClientProvider clusterclient.UserClusterConnectionProvider
 	dcs                   map[string]provider.DatacenterMeta
 }
 
@@ -434,7 +435,7 @@ func (r *testRunner) executeGinkgoRunWithRetries(log *logrus.Entry, run *ginkgoR
 
 func (r *testRunner) setupNodes(log *logrus.Entry, scenarioName string, cluster *kubermaticv1.Cluster, clusterKubeClient kubernetes.Interface, apiNodes []*kubermaticapiv1.Node, dc provider.DatacenterMeta) error {
 	log.Info("Creating machines...")
-	kubeMachineClient, err := r.clusterClientProvider.GetMachineClient(cluster)
+	client, err := r.clusterClientProvider.GetDynamicClient(cluster)
 	if err != nil {
 		return fmt.Errorf("failed to get the machine client for the cluster: %v", err)
 	}
@@ -455,11 +456,12 @@ func (r *testRunner) setupNodes(log *logrus.Entry, scenarioName string, cluster
 		}
 		// Make sure all nodes have different names across all scenarios - otherwise the Kubelet might not come up (OpenStack has this...)
 		m.Name = fmt.Sprintf("%s-machine-%d", scenarioName, i)
+		m.Namespace = metav1.NamespaceSystem
 		m.Spec.Name = strings.Replace(fmt.Sprintf("%s-node-%d", scenarioName, i), ".", "-", -1)
 
 		err = retryNAttempts(defaultAPIRetries, func(attempt int) error {
 			machineLog := log.WithFields(logrus.Fields{"machine": m.Name})
-			_, err := kubeMachineClient.ClusterV1alpha1().Machines(metav1.NamespaceSystem).Create(m)
+			err := client.Create(context.TODO(), m)
 			if err != nil {
 				if kerrors.IsAlreadyExists(err) {
 					return nil

api/cmd/kubermatic-api/main.go

@@ -114,9 +114,14 @@ func createInitProviders(options serverRunOptions) (providers, error) {
 			kubermaticSeedInformerFactory := kubermaticinformers.NewSharedInformerFactory(kubermaticSeedClient, informer.DefaultInformerResyncPeriod)
 			defaultImpersonationClientForSeed := kubernetesprovider.NewKubermaticImpersonationClient(cfg)
 
+			userClusterConnectionProvider, err := client.NewExternal(kubeInformerFactory.Core().V1().Secrets().Lister())
+			if err != nil {
+				return providers{}, fmt.Errorf("failed to get userClusterConnectionProvider: %v", err)
+			}
+
 			clusterProviders[ctx] = kubernetesprovider.NewClusterProvider(
 				defaultImpersonationClientForSeed.CreateImpersonatedClientSet,
-				client.NewExternal(kubeInformerFactory.Core().V1().Secrets().Lister()),
+				userClusterConnectionProvider,
 				kubermaticSeedInformerFactory.Kubermatic().V1().Clusters().Lister(),
 				options.workerName,
 				rbac.ExtractGroupPrefix,

api/cmd/kubermatic-controller-manager/main.go

@@ -241,11 +241,17 @@ func newControllerContext(
 	ctrlCtx.kubermaticInformerFactory = kubermaticinformers.NewFilteredSharedInformerFactory(ctrlCtx.kubermaticClient, informer.DefaultInformerResyncPeriod, metav1.NamespaceAll, selector)
 	ctrlCtx.kubeInformerFactory = kubeinformers.NewSharedInformerFactory(ctrlCtx.kubeClient, informer.DefaultInformerResyncPeriod)
 
-	var clientProvider *client.Provider
+	var clientProvider client.UserClusterConnectionProvider
 	if ctrlCtx.runOptions.kubeconfig != "" {
-		clientProvider = client.NewExternal(ctrlCtx.kubeInformerFactory.Core().V1().Secrets().Lister())
+		clientProvider, err = client.NewExternal(ctrlCtx.kubeInformerFactory.Core().V1().Secrets().Lister())
+		if err != nil {
+			return nil, fmt.Errorf("failed to get clientProvider: %v", err)
+		}
 	} else {
-		clientProvider = client.NewInternal(ctrlCtx.kubeInformerFactory.Core().V1().Secrets().Lister())
+		clientProvider, err = client.NewInternal(ctrlCtx.kubeInformerFactory.Core().V1().Secrets().Lister())
+		if err != nil {
+			return nil, fmt.Errorf("failed to get clientProvider: %v", err)
+		}
 	}
 	ctrlCtx.clientProvider = clientProvider
 

api/cmd/kubermatic-controller-manager/options.go

@@ -211,6 +211,6 @@ type controllerContext struct {
 	dynamicClient             ctrlruntimeclient.Client
 	dynamicCache              ctrlruntimecache.Cache
 	mgr                       manager.Manager
-	clientProvider            *client.Provider
+	clientProvider            client.UserClusterConnectionProvider
 	dcs                       map[string]provider.DatacenterMeta
 }

api/hack/run-rbac-generator.sh

@@ -11,6 +11,7 @@ KUBERMATIC_WORKERNAME=${KUBERMATIC_WORKERNAME:-$(uname -n)}
 cd $(go env GOPATH)/src/github.com/kubermatic/kubermatic/api
 ./_build/rbac-generator \
   -kubeconfig=../../secrets/seed-clusters/dev.kubermatic.io/kubeconfig \
+  -internal-address=127.0.0.1:8086 \
   -worker-name="$(tr -cd '[:alnum:]' <<< $KUBERMATIC_WORKERNAME | tr '[:upper:]' '[:lower:]')" \
   -logtostderr=1 \
   -v=6 $@

api/hack/run-userclustercontroller.sh

@@ -23,5 +23,6 @@ fi
 ./_build/user-cluster-controller-manager \
     -kubeconfig=$KUBECONFIG_USERCLUSTER_CONTROLLER \
     -logtostderr \
+    -internal-address=127.0.0.1:8087 \
     -v=4 \
     $ARGS

api/pkg/cluster/client/client.go

@@ -16,34 +16,48 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	aggregationclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
 
-	clusterv1alpha1clientset "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset"
-
+	clusterv1alpha1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
 	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 )
 
+// UserClusterConnectionProvider describes the interface available for accessing
+// resources inside the user cluster
+type UserClusterConnectionProvider interface {
+	GetClient(*kubermaticv1.Cluster, ...ConfigOption) (kubernetes.Interface, error)
+	GetApiextensionsClient(*kubermaticv1.Cluster, ...ConfigOption) (apiextensionsclientset.Interface, error)
+	GetAdmissionRegistrationClient(*kubermaticv1.Cluster, ...ConfigOption) (admissionregistrationclientset.AdmissionregistrationV1beta1Interface, error)
+	GetKubeAggregatorClient(*kubermaticv1.Cluster, ...ConfigOption) (aggregationclientset.Interface, error)
+	GetDynamicClient(*kubermaticv1.Cluster, ...ConfigOption) (ctrlruntimeclient.Client, error)
+	GetAdminKubeconfig(*kubermaticv1.Cluster) ([]byte, error)
+}
+
 // NewInternal returns a new instance of the client connection provider that
 // only works from within the seed cluster but has the advantage that it doesn't leave
 // the seed clusters network
-func NewInternal(secretLister corev1lister.SecretLister) *Provider {
-	return &Provider{secretLister: secretLister, useExternalAddress: false}
+func NewInternal(secretLister corev1lister.SecretLister) (UserClusterConnectionProvider, error) {
+	if err := clusterv1alpha1.SchemeBuilder.AddToScheme(scheme.Scheme); err != nil {
+		return nil, fmt.Errorf("failed to add clusterv1alpha1 to scheme: %v", err)
+	}
+	return &provider{secretLister: secretLister, useExternalAddress: false}, nil
 }
 
 // NewExternal returns a new instance of the client connection provider that
 // that uses the external cluster address and hence works from everywhere.
 // Use NewInternal if possible
-func NewExternal(secretLister corev1lister.SecretLister) *Provider {
-	return &Provider{secretLister: secretLister, useExternalAddress: true}
+func NewExternal(secretLister corev1lister.SecretLister) (UserClusterConnectionProvider, error) {
+	if err := clusterv1alpha1.AddToScheme(scheme.Scheme); err != nil {
+		return nil, fmt.Errorf("failed to add clusterv1alpha1 to scheme: %v", err)
+	}
+	return &provider{secretLister: secretLister, useExternalAddress: true}, nil
 }
 
-// Provider offers functions to interact with a user cluster
-type Provider struct {
+type provider struct {
 	secretLister       corev1lister.SecretLister
 	useExternalAddress bool
 }
 
 // GetAdminKubeconfig returns the admin kubeconfig for the given cluster
-func (p *Provider) GetAdminKubeconfig(c *kubermaticv1.Cluster) ([]byte, error) {
+func (p *provider) GetAdminKubeconfig(c *kubermaticv1.Cluster) ([]byte, error) {
 	var s *corev1.Secret
 	var err error
 	if p.useExternalAddress {
@@ -67,7 +81,7 @@ func (p *Provider) GetAdminKubeconfig(c *kubermaticv1.Cluster) ([]byte, error) {
 type ConfigOption func(*restclient.Config) *restclient.Config
 
 // GetClientConfig returns the client config used for initiating a connection for the given cluster
-func (p *Provider) GetClientConfig(c *kubermaticv1.Cluster, options ...ConfigOption) (*restclient.Config, error) {
+func (p *provider) GetClientConfig(c *kubermaticv1.Cluster, options ...ConfigOption) (*restclient.Config, error) {
 	b, err := p.GetAdminKubeconfig(c)
 	if err != nil {
 		return nil, err
@@ -103,7 +117,7 @@ func (p *Provider) GetClientConfig(c *kubermaticv1.Cluster, options ...ConfigOpt
 }
 
 // GetClient returns a kubernetes client to interact with the given cluster
-func (p *Provider) GetClient(c *kubermaticv1.Cluster, options ...ConfigOption) (kubernetes.Interface, error) {
+func (p *provider) GetClient(c *kubermaticv1.Cluster, options ...ConfigOption) (kubernetes.Interface, error) {
 	config, err := p.GetClientConfig(c, options...)
 	if err != nil {
 		return nil, err
@@ -117,22 +131,8 @@ func (p *Provider) GetClient(c *kubermaticv1.Cluster, options ...ConfigOption) (
 	return client, nil
 }
 
-// GetMachineClient returns a client to interact with machine resources for the given cluster
-func (p *Provider) GetMachineClient(c *kubermaticv1.Cluster, options ...ConfigOption) (clusterv1alpha1clientset.Interface, error) {
-	config, err := p.GetClientConfig(c, options...)
-	if err != nil {
-		return nil, err
-	}
-	client, err := clusterv1alpha1clientset.NewForConfig(config)
-	if err != nil {
-		return nil, err
-	}
-
-	return client, nil
-}
-
 // GetApiextensionsClient returns a client to interact with apiextension resources for the given cluster
-func (p *Provider) GetApiextensionsClient(c *kubermaticv1.Cluster, options ...ConfigOption) (apiextensionsclientset.Interface, error) {
+func (p *provider) GetApiextensionsClient(c *kubermaticv1.Cluster, options ...ConfigOption) (apiextensionsclientset.Interface, error) {
 	config, err := p.GetClientConfig(c, options...)
 	if err != nil {
 		return nil, err
@@ -141,7 +141,7 @@ func (p *Provider) GetApiextensionsClient(c *kubermaticv1.Cluster, options ...Co
 }
 
 // GetAdmissionRegistrationClient returns a client to interact with admissionregistration resources
-func (p *Provider) GetAdmissionRegistrationClient(c *kubermaticv1.Cluster, options ...ConfigOption) (admissionregistrationclientset.AdmissionregistrationV1beta1Interface, error) {
+func (p *provider) GetAdmissionRegistrationClient(c *kubermaticv1.Cluster, options ...ConfigOption) (admissionregistrationclientset.AdmissionregistrationV1beta1Interface, error) {
 	config, err := p.GetClientConfig(c, options...)
 	if err != nil {
 		return nil, err
@@ -150,7 +150,7 @@ func (p *Provider) GetAdmissionRegistrationClient(c *kubermaticv1.Cluster, optio
 }
 
 // GetKubeAggregatorClient returns a client to interact with the aggregation API for the given cluster
-func (p *Provider) GetKubeAggregatorClient(c *kubermaticv1.Cluster, options ...ConfigOption) (aggregationclientset.Interface, error) {
+func (p *provider) GetKubeAggregatorClient(c *kubermaticv1.Cluster, options ...ConfigOption) (aggregationclientset.Interface, error) {
 	config, err := p.GetClientConfig(c, options...)
 	if err != nil {
 		return nil, err
@@ -159,17 +159,12 @@ func (p *Provider) GetKubeAggregatorClient(c *kubermaticv1.Cluster, options ...C
 }
 
 // GetDynamicClient returns a dynamic client
-func (p *Provider) GetDynamicClient(c *kubermaticv1.Cluster, options ...ConfigOption) (ctrlruntimeclient.Client, error) {
+func (p *provider) GetDynamicClient(c *kubermaticv1.Cluster, options ...ConfigOption) (ctrlruntimeclient.Client, error) {
 	config, err := p.GetClientConfig(c, options...)
 	if err != nil {
 		return nil, err
 	}
-	combinedScheme := scheme.Scheme
-	mapper, err := apiutil.NewDiscoveryRESTMapper(config)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create rest mapper: %v", err)
-	}
-	dynamicClient, err := ctrlruntimeclient.New(config, ctrlruntimeclient.Options{Scheme: combinedScheme, Mapper: mapper})
+	dynamicClient, err := ctrlruntimeclient.New(config, ctrlruntimeclient.Options{})
 	if err != nil {
 		return nil, fmt.Errorf("failed to create dynamic client: %v", err)
 	}

api/pkg/controller/cluster/cluster_controller.go

@@ -44,15 +44,13 @@ import (
 	"k8s.io/client-go/util/retry"
 	"k8s.io/client-go/util/workqueue"
 
-	clusterv1alpha1clientset "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset"
 	ctrlruntimecache "sigs.k8s.io/controller-runtime/pkg/cache"
 	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // userClusterConnectionProvider offers functions to retrieve clients for the given user clusters
 type userClusterConnectionProvider interface {
 	GetClient(*kubermaticv1.Cluster, ...k8cuserclusterclient.ConfigOption) (kubernetes.Interface, error)
-	GetMachineClient(*kubermaticv1.Cluster, ...k8cuserclusterclient.ConfigOption) (clusterv1alpha1clientset.Interface, error)
 	GetApiextensionsClient(*kubermaticv1.Cluster, ...k8cuserclusterclient.ConfigOption) (apiextensionsclientset.Interface, error)
 	GetAdmissionRegistrationClient(*kubermaticv1.Cluster, ...k8cuserclusterclient.ConfigOption) (admissionregistrationclientset.AdmissionregistrationV1beta1Interface, error)
 	GetKubeAggregatorClient(*kubermaticv1.Cluster, ...k8cuserclusterclient.ConfigOption) (aggregationclientset.Interface, error)

api/pkg/controller/cluster/cluster_controller_test.go

@@ -31,6 +31,10 @@ func newTestController(kubeObjects []runtime.Object, kubermaticObjects []runtime
 
 	kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, time.Minute*5)
 	kubermaticInformerFactory := kubermaticinformers.NewSharedInformerFactory(kubermaticClient, time.Minute*5)
+	client, err := client.NewInternal(kubeInformerFactory.Core().V1().Secrets().Lister())
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	dynamicClient := ctrlruntimefakeclient.NewFakeClient()
 	controller, err := NewController(
@@ -40,7 +44,7 @@ func newTestController(kubeObjects []runtime.Object, kubermaticObjects []runtime
 		TestExternalURL,
 		TestDC,
 		dcs,
-		client.NewInternal(kubeInformerFactory.Core().V1().Secrets().Lister()),
+		client,
 		"",
 		"",
 		"192.0.2.0/24",