-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: Jan Lohage <lohage@23technologies.cloud> Signed-off-by: Jens Schneider <jens.schneider.ac@posteo.de>
- Loading branch information
Showing
16 changed files
with
465 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
/* | ||
Copyright © 2022 NAME HERE <EMAIL ADDRESS> | ||
*/ | ||
package cmd | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"github.com/23technologies/23kectl/pkg/check" | ||
"github.com/fluxcd/helm-controller/api/v2beta1" | ||
"github.com/fluxcd/kustomize-controller/api/v1beta2" | ||
"github.com/spf13/cobra" | ||
"sigs.k8s.io/controller-runtime/pkg/client" | ||
) | ||
|
||
// installCmd represents the install command | ||
var doctorCmd = &cobra.Command{ | ||
Use: "doctor", | ||
Short: "Check the status of a current 23ke installation", | ||
Long: `This command will print status messages for flux resources. | ||
If e.g. a HelmRelease failed, the error message message including a hint | ||
will be printed. | ||
`, | ||
RunE: func(cmd *cobra.Command, args []string) error { | ||
|
||
doctor() | ||
return nil | ||
}, | ||
} | ||
|
||
func init() { | ||
rootCmd.AddCommand(doctorCmd) | ||
doctorCmd.PersistentFlags().String("kubeconfig", "", "The KUBECONFIG of your base cluster") | ||
} | ||
|
||
func doctor() { | ||
var checks []check.Check | ||
|
||
hrList := &v2beta1.HelmReleaseList{} | ||
_ = check.KubeClient.List(context.TODO(), hrList, &client.ListOptions{Namespace: "flux-system"}) | ||
|
||
for _, hr := range hrList.Items { | ||
checks = append(checks, &check.HelmReleaseCheck{Name: hr.Name, Namespace: hr.Namespace}) | ||
} | ||
|
||
ksList := &v1beta2.KustomizationList{} | ||
_ = check.KubeClient.List(context.TODO(), ksList, &client.ListOptions{Namespace: "flux-system"}) | ||
|
||
for _, ks := range ksList.Items { | ||
checks = append(checks, &check.KustomizationCheck{Name: ks.Name, Namespace: ks.Namespace}) | ||
} | ||
|
||
fmt.Print("\033[H\033[2J") | ||
|
||
for _, c := range checks { | ||
result := c.Run() | ||
|
||
emoji := "⌛" | ||
|
||
if result.IsError { | ||
emoji = "❌" | ||
} else if result.IsOkay { | ||
emoji = "✔️" | ||
} | ||
|
||
fmt.Printf("%s %s status: %s\n", emoji, c.GetName(), result.Status) | ||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
package check | ||
|
||
type Runnable interface { | ||
Run() *Result | ||
} | ||
|
||
type WithHint interface { | ||
Hint() string | ||
} | ||
|
||
type WithFix interface { | ||
Fix() | ||
} | ||
|
||
type WithOnError interface { | ||
OnError() | ||
} | ||
|
||
type WithName interface { | ||
GetName() string | ||
} | ||
|
||
type Check interface { | ||
Runnable | ||
WithName | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
package check | ||
|
||
import ( | ||
"context" | ||
v1 "github.com/fluxcd/source-controller/api/v1beta2" | ||
"sigs.k8s.io/controller-runtime/pkg/client" | ||
) | ||
|
||
type HelmChartsCheck struct { | ||
Name string | ||
Namespace string | ||
} | ||
|
||
func (d *HelmChartsCheck) GetName() string { | ||
return d.Name | ||
} | ||
|
||
func (d *HelmChartsCheck) Run() *Result { | ||
result := &Result{} | ||
|
||
hc := &v1.HelmChart{} | ||
|
||
err := KubeClient.Get(context.Background(), client.ObjectKey{ | ||
Namespace: d.Namespace, | ||
Name: d.Name, | ||
}, hc) | ||
|
||
if err != nil { | ||
result.IsError = true | ||
return result | ||
} | ||
|
||
result.Status = getMessage(hc.Status.Conditions, "Ready") | ||
|
||
if result.Status == "Applied revision" { | ||
result.IsError = false | ||
result.IsOkay = true | ||
} else if result.Status == "SOME DEFINITIVE ERROR" { | ||
result.IsError = true | ||
result.IsOkay = false | ||
} | ||
|
||
return result | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,152 @@ | ||
package check | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"regexp" | ||
"strings" | ||
|
||
helmv2 "github.com/fluxcd/helm-controller/api/v2beta1" | ||
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" | ||
"github.com/mitchellh/go-wordwrap" | ||
corev1 "k8s.io/api/core/v1" | ||
"sigs.k8s.io/controller-runtime/pkg/client" | ||
) | ||
|
||
type HelmReleaseCheck struct { | ||
Name string | ||
Namespace string | ||
} | ||
|
||
func (d *HelmReleaseCheck) GetName() string { | ||
return d.Name | ||
} | ||
|
||
func (d *HelmReleaseCheck) Run() *Result { | ||
result := &Result{} | ||
|
||
hr := &helmv2.HelmRelease{} | ||
|
||
err := KubeClient.Get(context.Background(), client.ObjectKey{ | ||
Namespace: d.Namespace, | ||
Name: d.Name, | ||
}, hr) | ||
|
||
if err != nil { | ||
result.IsError = true | ||
return result | ||
} | ||
|
||
// define a slice of handler including a regexp and a function | ||
// if we find a match we process the event by an appropriate function | ||
// this is assumed to stay branchless in the future which enables easy extensibility | ||
// the order of processing is important as we prioritize the status messages | ||
type handler struct { | ||
regex *regexp.Regexp | ||
fn func(res *Result, matches []string) | ||
} | ||
|
||
handlers := []handler{ | ||
{ | ||
regex: regexp.MustCompile("Helm test failed: pod (?P<podName>.*) failed"), | ||
fn: handeHelmTestError, | ||
}, | ||
{ | ||
regex: regexp.MustCompile("(install retries exhausted|upgrade retries exhausted|Helm install failed|Helm upgrade failed).*"), | ||
fn: func(res *Result, matches []string) { | ||
res.Status = prettify(matches[0]) | ||
res.IsError = true | ||
res.IsOkay = false | ||
}, | ||
}, | ||
{ | ||
regex: regexp.MustCompile("^HelmChart '(?P<namespace>.*)/(?P<name>.*)' is not ready$"), | ||
fn: handleHelmChartError, | ||
}, | ||
{ | ||
regex: regexp.MustCompile("Release reconciliation succeeded"), | ||
fn: func(res *Result, matches []string) { | ||
res.Status = prettify(matches[0]) | ||
res.IsError = false | ||
res.IsOkay = true | ||
}, | ||
}, | ||
} | ||
|
||
// iterate over status conditions in the helm releases | ||
// here all useful information about potential errors should be found | ||
for _, curHandler := range handlers { | ||
for _, condition := range hr.GetConditions() { | ||
matches := curHandler.regex.FindStringSubmatch(condition.Message) | ||
if matches != nil { | ||
curHandler.fn(result, matches) | ||
return result | ||
} | ||
} | ||
} | ||
|
||
return result | ||
} | ||
|
||
// handeHelmTestError ... | ||
func handeHelmTestError(res *Result, matches []string) { | ||
|
||
// It seems controller-runtime does not allow to access the logs. | ||
// Use kubectl directly for the moment. | ||
test := KubeClientGo.CoreV1().Pods("garden").GetLogs(matches[1], &corev1.PodLogOptions{}) | ||
logs, err := test.Do(context.Background()).Raw() | ||
log := string(logs) | ||
if err != nil { | ||
log = fmt.Sprintf("couldn't get pod logs: %s", err) | ||
} | ||
|
||
// Do some easy formatting for the moment. | ||
// We should definitely look for some package doing the job in the end. | ||
const replacement = "\n > " | ||
var replacer = strings.NewReplacer( | ||
"\r\n", replacement, | ||
"\r", replacement, | ||
"\n", replacement, | ||
"\v", replacement, | ||
"\f", replacement, | ||
"\u0085", replacement, | ||
"\u2028", replacement, | ||
"\u2029", replacement, | ||
) | ||
|
||
newline := "\n > " | ||
res.Status = matches[0] + newline + replacer.Replace(wordwrap.WrapString(strings.TrimSpace(log), 100)) | ||
|
||
res.IsError = true | ||
res.IsOkay = false | ||
} | ||
|
||
func handleHelmChartError(res *Result, matches []string) { | ||
namespace := matches[1] | ||
name := matches[2] | ||
|
||
hc := &sourcev1.HelmChart{} | ||
|
||
err := KubeClient.Get(context.Background(), client.ObjectKey{ | ||
Namespace: namespace, | ||
Name: name, | ||
}, hc) | ||
|
||
status := matches[0] | ||
|
||
if err != nil { | ||
status = status + ": " + err.Error() | ||
} else { | ||
hcReadyMessage := getMessage(hc.Status.Conditions, "Ready") | ||
status = status + ": " + hcReadyMessage | ||
} | ||
|
||
res.Status = prettify(status) | ||
res.IsError = true | ||
res.IsOkay = false | ||
} | ||
|
||
func prettify(message string) string { | ||
newline := "\n > " | ||
return strings.Replace(message, ": ", newline, -1) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
package check | ||
|
||
import ( | ||
"context" | ||
"github.com/fluxcd/kustomize-controller/api/v1beta2" | ||
"sigs.k8s.io/controller-runtime/pkg/client" | ||
"strings" | ||
) | ||
|
||
type KustomizationCheck struct { | ||
Name string | ||
Namespace string | ||
} | ||
|
||
func (d *KustomizationCheck) GetName() string { | ||
return d.Name | ||
} | ||
|
||
func (d *KustomizationCheck) Run() *Result { | ||
result := &Result{} | ||
|
||
ks := &v1beta2.Kustomization{} | ||
|
||
err := KubeClient.Get(context.Background(), client.ObjectKey{ | ||
Namespace: d.Namespace, | ||
Name: d.Name, | ||
}, ks) | ||
|
||
if err != nil { | ||
result.IsError = true | ||
return result | ||
} | ||
|
||
result.Status = getMessage(ks.Status.Conditions, "Ready") | ||
|
||
if strings.Contains(result.Status, "Applied revision") { | ||
result.IsError = false | ||
result.IsOkay = true | ||
} else if result.Status == "SOME DEFINITIVE ERROR" { | ||
result.IsError = true | ||
result.IsOkay = false | ||
} | ||
|
||
return result | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
package check | ||
|
||
type Result struct { | ||
IsError bool | ||
IsOkay bool | ||
Status string | ||
} |
Oops, something went wrong.