feat: use pre-downloaded binaries for cfssl (#568)

This PR makes sure that the init pod will use pre-downloaded binaries,
so that the operator can be deployed in offline scenarios. Also it no
longer makes the assumption that binaries like `sh` and `chmod` exist in
the docker image (a lot of hardened/ minimal images don't have shell on
them).

Other changes:
- modified `server.pem` and `server-key.pem` to `ca.pem` and
`ca-key.pem` - AFAICT, these are the file names generated by cfssljson ;
yes I know it's confusing but these are different than the input ones.
- passed additional argument "-r" to the init pod - we don't want it
failing on subsequent deployments/ if a webhook already exists.

Fixes #567.

Author: virgilp

src/KubeOps.Testing/KubernetesOperatorFactory.cs

@@ -42,6 +42,13 @@ public KubernetesOperatorFactory<TTestStartup> WithSolutionRelativeContentRoot(s
         return this;
     }
 
+    public IManagedResourceController GetController<TEntity>()
+        where TEntity : IKubernetesObject<V1ObjectMeta> =>
+            Services
+            .GetRequiredService<IControllerInstanceBuilder>()
+            .BuildControllers<TEntity>()
+            .First();
+
     /// <summary>
     /// Start the server.
     /// </summary>

src/KubeOps/Operator/Commands/CommandHelpers/CertificateGenerator.cs

@@ -1,11 +1,10 @@
 using System.Diagnostics;
-using System.Runtime.InteropServices;
-using McMaster.Extensions.CommandLineUtils;
 
 namespace KubeOps.Operator.Commands.CommandHelpers;
 
 internal class CertificateGenerator : IDisposable
 {
+    /* Suggested URLs for downloading the executables into the docker image
     private const string CfsslUrlWindows =
         "https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssl_1.5.0_windows_amd64.exe";
 
@@ -23,6 +22,7 @@ internal class CertificateGenerator : IDisposable
 
     private const string CfsslJsonUrlMacOs =
         "https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssljson_1.5.0_darwin_amd64";
+    */
 
     private const string CaConfig =
         @"{""signing"":{""default"":{""expiry"":""43800h""},""profiles"":{""server"":{""expiry"":""43800h"",""usages"":[""signing"",""key encipherment"",""server auth""]}}}}";
@@ -34,21 +34,17 @@ internal class CertificateGenerator : IDisposable
     private readonly string _tempDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
 
     private bool _initialized;
-    private string? _cfssl;
-    private string? _cfssljson;
     private string? _caconfig;
     private string? _cacsr;
     private string? _servercsr;
+    private string _cfssl = "cfssl";
+    private string _cfssljson = "cfssljson";
 
     public CertificateGenerator(TextWriter appOut)
     {
         _appOut = appOut;
     }
 
-    private static string ShellExecutor => RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-        ? "cmd.exe"
-        : "/bin/sh";
-
     public void Dispose()
     {
         Delete(_caconfig);
@@ -62,6 +58,9 @@ public async Task CreateCaCertificateAsync(string outputFolder)
     {
         if (!_initialized)
         {
+            var cfsslFolder = Environment.GetEnvironmentVariable("CFSSL_EXECUTABLES_PATH") ?? "/operator";
+            _cfssl = $"{cfsslFolder}/{_cfssl}";
+            _cfssljson = $"{cfsslFolder}/{_cfssljson}";
             await PrepareExecutables();
             _initialized = true;
         }
@@ -70,18 +69,8 @@ public async Task CreateCaCertificateAsync(string outputFolder)
 
         await _appOut.WriteLineAsync($@"Generating certificates to ""{outputFolder}"".");
         await _appOut.WriteLineAsync("Generating CA certificate.");
-        await ExecuteProcess(
-            new Process
-            {
-                StartInfo = new ProcessStartInfo
-                {
-                    WorkingDirectory = outputFolder,
-                    FileName = ShellExecutor,
-                    Arguments = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-                        ? $"/c {_cfssl} gencert -initca {_cacsr} | {_cfssljson} -bare ca -"
-                        : $@"-c ""{_cfssl} gencert -initca {_cacsr} | {_cfssljson} -bare ca -""",
-                },
-            });
+
+        await GenCertAsync($"-initca {_cacsr}", outputFolder);
 
         await ListDir(outputFolder);
     }
@@ -110,33 +99,15 @@ await serverCsrStream.WriteLineAsync(
 
         await _appOut.WriteLineAsync(
             $@"Generating server certificate for ""{name}"" in namespace ""{@namespace}"".");
-        await ExecuteProcess(
-            new Process
-            {
-                StartInfo = new ProcessStartInfo
-                {
-                    WorkingDirectory = outputFolder,
-                    FileName = ShellExecutor,
-                    Arguments = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-                        ? $"/c {_cfssl} gencert -ca=file:{caPath.Replace('\\', '/')} -ca-key=file:{caKeyPath.Replace('\\', '/')} -config={_caconfig} -profile=server {_servercsr} | {_cfssljson} -bare server -"
-                        : $@"-c ""{_cfssl} gencert -ca={caPath} -ca-key={caKeyPath} -config={_caconfig} -profile=server {_servercsr} | {_cfssljson} -bare server -""",
-                },
-            });
-
+        var arguments =
+            $"-ca=file:{caPath.Replace('\\', '/')} -ca-key=file:{caKeyPath.Replace('\\', '/')} -config={_caconfig} -profile=server {_servercsr}";
+        await GenCertAsync(arguments, outputFolder);
         await ListDir(outputFolder);
     }
 
     private static string ServerCsr(params string[] serverNames) =>
         $@"{{""CN"":""Operator Service"",""hosts"":[""{string.Join(@""",""", serverNames)}""],""key"":{{""algo"":""ecdsa"",""size"":256}},""names"":[{{""C"":""DEV"",""L"":""Kubernetes""}}]}}";
 
-    private static Task ExecuteProcess(Process process)
-        => Task.Run(
-            () =>
-            {
-                process.Start();
-                process.WaitForExit(2000);
-            });
-
     private static void Delete(string? file)
     {
         if (file == null)
@@ -150,66 +121,50 @@ private static void Delete(string? file)
         }
     }
 
+    private async Task GenCertAsync(
+        string arguments,
+        string outputFolder,
+        CancellationToken cancellationToken = default)
+    {
+        var genCertPsi = new ProcessStartInfo
+        {
+            WorkingDirectory = outputFolder,
+            FileName = _cfssl,
+            Arguments = $"gencert {arguments}",
+            RedirectStandardOutput = true,
+            UseShellExecute = false,
+        };
+        var writeJsonPsi = new ProcessStartInfo
+        {
+            WorkingDirectory = outputFolder,
+            FileName = _cfssljson,
+            Arguments = "-bare ca -",
+            RedirectStandardInput = true,
+            UseShellExecute = false,
+        };
+        using var genCert = new Process { StartInfo = genCertPsi };
+        using var writeJson = new Process { StartInfo = writeJsonPsi };
+
+        genCert.Start();
+        await genCert.WaitForExitAsync(cancellationToken);
+
+        writeJson.Start();
+        await writeJson.StandardInput.WriteAsync(await genCert.StandardOutput.ReadToEndAsync());
+        await writeJson.StandardInput.FlushAsync();
+        writeJson.StandardInput.Close();
+        await writeJson.WaitForExitAsync(cancellationToken);
+    }
+
     private async Task PrepareExecutables()
     {
         Directory.CreateDirectory(_tempDirectory);
-        _cfssl = Path.Join(_tempDirectory, Path.GetRandomFileName());
-        _cfssljson = Path.Join(_tempDirectory, Path.GetRandomFileName());
         _caconfig = Path.Join(_tempDirectory, Path.GetRandomFileName());
         _cacsr = Path.Join(_tempDirectory, Path.GetRandomFileName());
 
-        using (var client = new HttpClient())
-        await using (var cfsslStream = new FileStream(_cfssl, FileMode.CreateNew))
-        await using (var cfsslJsonStream = new FileStream(_cfssljson, FileMode.CreateNew))
-        await using (var caConfigStream = new StreamWriter(new FileStream(_caconfig, FileMode.CreateNew)))
-        await using (var caCsrStream = new StreamWriter(new FileStream(_cacsr, FileMode.CreateNew)))
-        {
-            await caConfigStream.WriteLineAsync(CaConfig);
-            await caCsrStream.WriteLineAsync(CaCsr);
-
-            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-            {
-                await _appOut.WriteLineAsync("Download cfssl / cfssljson for windows.");
-                await using var cfsslDl = await client.GetStreamAsync(CfsslUrlWindows);
-                await using var cfsslJsonDl = await client.GetStreamAsync(CfsslJsonUrlWindows);
-
-                await cfsslDl.CopyToAsync(cfsslStream);
-                await cfsslJsonDl.CopyToAsync(cfsslJsonStream);
-            }
-            else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
-            {
-                await _appOut.WriteLineAsync("Download cfssl / cfssljson for linux.");
-                await using var cfsslDl = await client.GetStreamAsync(CfsslUrlLinux);
-                await using var cfsslJsonDl = await client.GetStreamAsync(CfsslJsonUrlLinux);
-
-                await cfsslDl.CopyToAsync(cfsslStream);
-                await cfsslJsonDl.CopyToAsync(cfsslJsonStream);
-            }
-            else
-            {
-                await _appOut.WriteLineAsync("Download cfssl / cfssljson for macos.");
-                await using var cfsslDl = await client.GetStreamAsync(CfsslUrlMacOs);
-                await using var cfsslJsonDl = await client.GetStreamAsync(CfsslJsonUrlMacOs);
-
-                await cfsslDl.CopyToAsync(cfsslStream);
-                await cfsslJsonDl.CopyToAsync(cfsslJsonStream);
-            }
-        }
-
-        if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-        {
-            await _appOut.WriteLineAsync("Make unix binaries executable.");
-            await ExecuteProcess(
-                new Process
-                {
-                    StartInfo = new ProcessStartInfo
-                    {
-                        FileName = "chmod",
-                        Arguments = ArgumentEscaper.EscapeAndConcatenate(
-                            new[] { "+x", _cfssl, _cfssljson }),
-                    },
-                });
-        }
+        await using var caConfigStream = new StreamWriter(new FileStream(_caconfig, FileMode.CreateNew));
+        await using var caCsrStream = new StreamWriter(new FileStream(_cacsr, FileMode.CreateNew));
+        await caConfigStream.WriteLineAsync(CaConfig);
+        await caCsrStream.WriteLineAsync(CaCsr);
     }
 
     private async Task ListDir(string directory)

src/KubeOps/Operator/Commands/Generators/DockerGenerator.cs

@@ -56,6 +56,12 @@ public string GenerateDockerfile() =>
 WORKDIR /operator
 
 COPY ./ ./
+RUN curl -L -o cfssl https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssl_1.5.0_linux_amd64
+RUN curl -L -o cfssljson https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssljson_1.5.0_linux_amd64
+RUN chmod +x ./cfssl
+RUN chmod +x ./cfssljson
+RUN mkdir out
+RUN cp cfssl cfssljson out/
 RUN dotnet publish -c Release -o out {ProjectToBuild}
 
 # The runner for the application

src/KubeOps/Operator/Commands/Generators/OperatorGenerator.cs

@@ -53,8 +53,8 @@ public async Task<int> OnExecuteAsync(CommandLineApplication app)
                                 {
                                     $"KESTREL__ENDPOINTS__HTTP__URL=http://0.0.0.0:{_settings.HttpPort}",
                                     $"KESTREL__ENDPOINTS__HTTPS__URL=https://0.0.0.0:{_settings.HttpsPort}",
-                                    "KESTREL__ENDPOINTS__HTTPS__CERTIFICATE__PATH=/certs/server.pem",
-                                    "KESTREL__ENDPOINTS__HTTPS__CERTIFICATE__KEYPATH=/certs/server-key.pem",
+                                    "KESTREL__ENDPOINTS__HTTPS__CERTIFICATE__PATH=/certs/ca.pem",
+                                    "KESTREL__ENDPOINTS__HTTPS__CERTIFICATE__KEYPATH=/certs/ca-key.pem",
                                 },
                             },
                         }
@@ -112,7 +112,7 @@ public async Task<int> OnExecuteAsync(CommandLineApplication app)
                                         {
                                             Image = "operator",
                                             Name = "webhook-installer",
-                                            Args = new[] { "webhooks", "install", },
+                                            Args = new[] { "webhooks", "install", "-r" },
                                             Env = new List<V1EnvVar>
                                             {
                                                 new()

src/KubeOps/Operator/Commands/Management/Webhooks/Install.cs

@@ -131,6 +131,7 @@ await client.Create(
             if (existingItem != null)
             {
                 await app.Out.WriteLineAsync("Validator existed, updating.");
+                existingItem.Webhooks = validatorConfig.Webhooks;
                 await client.Update(existingItem);
             }
             else

src/KubeOps/Operator/Controller/IManagedResourceController.cs

@@ -1,6 +1,6 @@
 namespace KubeOps.Operator.Controller;
 
-internal interface IManagedResourceController : IDisposable
+public interface IManagedResourceController : IDisposable
 {
     Task StartAsync();
 

tests/KubeOps.TestOperator.Test/TestController.Test.cs

@@ -20,11 +20,7 @@ public TestControllerTest(KubernetesOperatorFactory<TestStartup> factory)
     {
         _factory = factory.WithSolutionRelativeContentRoot("tests/KubeOps.TestOperator");
 
-        _controller = _factory.Services
-            .GetRequiredService<IControllerInstanceBuilder>()
-            .BuildControllers<V1TestEntity>()
-            .First();
-
+        _controller = _factory.GetController<V1TestEntity>();
         _managerMock = _factory.Services.GetRequiredService<Mock<IManager>>();
         _managerMock.Reset();
     }