From 281963836c188484dc5d19939d44d9ea612e3a53 Mon Sep 17 00:00:00 2001 From: Jiaqi Luo <6218999+jiaqiluo@users.noreply.github.com> Date: Tue, 5 Apr 2022 09:57:41 -0700 Subject: [PATCH] This PR adds a new data structure AppDefault which maps an app's version range to a release range and can be used to set the default release version(s) per app version(s). This PR improves the handling of HTTP errors, especially the 404 not found, from the source URL. This PR also bumps github.com/urfave/cli/v2 to v2.4.0 to fix the issue where the default value is not set for string slice flags. --- README.md | 17 ++++++ channels.yaml | 82 ++++++++++++++++------------ go.mod | 2 +- go.sum | 12 ++-- main.go | 9 ++- pkg/config/config.go | 51 +++++++++++------ pkg/config/get.go | 41 ++++++++++++++ pkg/model/config.go | 14 +++++ pkg/server/server.go | 5 ++ pkg/server/store/appdefault/store.go | 30 ++++++++++ 10 files changed, 203 insertions(+), 60 deletions(-) create mode 100644 pkg/server/store/appdefault/store.go diff --git a/README.md b/README.md index 2d7fa79..d885b6c 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,23 @@ The primary usecase for this project is currently Rancher's [system-upgrade-cont This service is in proudction for k3s here: https://update.k3s.io/v1-release/channels, which is driven by this config: https://github.com/rancher/k3s/blob/master/channel.yaml. Each channel's `self` link is the URL that will resolve to the latest GitHub release page for that channel. +## Compile and Run +1. build the binary +``` +go build . +``` +2. run the server +``` + ./channelserver --config-key k3s --path-prefix v1-release --channel-server-version v2.3.1 +``` + +3. run the following curl command or open the url in the browser +``` +curl 0.0.0.0:8080/v1-release/release +curl 0.0.0.0:8080/v1-release/channel +curl 0.0.0.0:8080/v1-release/appdefault +``` + ## License Copyright (c) 2020 [Rancher Labs, Inc.](http://rancher.com) diff --git a/channels.yaml b/channels.yaml index df8e218..faa424a 100644 --- a/channels.yaml +++ b/channels.yaml @@ -1,35 +1,49 @@ # Example channels config -channels: -- name: stable - latest: v1.17.2+k3s1 -- name: latest - latestRegexp: .* - excludeRegexp: rc -- name: testing - latestRegexp: .* -- name: v1.16 - latestRegexp: v1\.16\..* - excludeRegexp: rc -- name: v1.16-testing - latestRegexp: v1\.16\.* -- name: v1.17 - latestRegexp: v1\.17\..* - excludeRegexp: rc -- name: v1.17-testing - latestRegexp: v1\.17\..* -- name: v1.0 - latestRegexp: v1\.0\..* - excludeRegexp: rc -- name: v1.0-testing - latestRegexp: v1\.0\..* -releases: -- version: v1.15.3+k3s2 - minChannelServerVersion: v2.3.0 - maxChannelServerVersion: v2.3.5 -- version: v1.17.4+k3s1 - minChannelServerVersion: v2.4.0 - maxChannelServerVersion: v2.4.5 -github: - owner: rancher - repo: k3s -redirectBase: https://github.com/rancher/k3s/releases/tag/ +k3s: + channels: + - name: stable + latest: v1.17.2+k3s1 + - name: latest + latestRegexp: .* + excludeRegexp: rc + - name: testing + latestRegexp: .* + - name: v1.16 + latestRegexp: v1\.16\..* + excludeRegexp: rc + - name: v1.16-testing + latestRegexp: v1\.16\.* + - name: v1.17 + latestRegexp: v1\.17\..* + excludeRegexp: rc + - name: v1.17-testing + latestRegexp: v1\.17\..* + - name: v1.0 + latestRegexp: v1\.0\..* + excludeRegexp: rc + - name: v1.0-testing + latestRegexp: v1\.0\..* + releases: + - version: v1.15.3+k3s2 + minChannelServerVersion: v2.3.0 + maxChannelServerVersion: v2.3.5 + - version: v1.17.4+k3s1 + minChannelServerVersion: v2.4.0 + maxChannelServerVersion: v2.4.5 + appDefaults: + - appName: rancher + defaults: + - appVersion: '> 2.6.2-0 < 2.6.3-0' + defaultVersion: '1.20.x' + - appVersion: '> 2.6.2-0 < 2.6.4-0' + defaultVersion: '1.21.x' + - appVersion: '> 2.6.4-0' + defaultVersion: '1.22.x' + - appName: anotherApp + defaults: + - appVersion: '1.2.x' + defaultVersion: '1.19.5' + github: + owner: rancher + repo: k3s + redirectBase: https://github.com/rancher/k3s/releases/tag/ diff --git a/go.mod b/go.mod index d643fa5..68e1a66 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,6 @@ require ( github.com/rancher/apiserver v0.0.0-20201023000256-1a0a904f9197 github.com/rancher/wrangler v0.8.11-0.20220120160420-18c996a8e956 github.com/sirupsen/logrus v1.6.0 - github.com/urfave/cli/v2 v2.2.0 + github.com/urfave/cli/v2 v2.4.0 sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index 7e17153..ce5ab32 100644 --- a/go.sum +++ b/go.sum @@ -80,9 +80,9 @@ github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -382,11 +382,11 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -427,8 +427,8 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= -github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/urfave/cli/v2 v2.4.0 h1:m2pxjjDFgDxSPtO8WSdbndj17Wu2y8vOT86wE/tjr+I= +github.com/urfave/cli/v2 v2.4.0/go.mod h1:NX9W0zmTvedE5oDoOMs2RTC8RvdK98NTYZE5LbaEYPg= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= diff --git a/main.go b/main.go index d536eb6..f95c78a 100644 --- a/main.go +++ b/main.go @@ -22,6 +22,7 @@ var ( SubKeys cli.StringSlice ChannelServerVersion string PathPrefix cli.StringSlice + AppName string ) func main() { @@ -56,6 +57,12 @@ func main() { EnvVars: []string{"CHANNEL_SERVER_VERSION"}, Destination: &ChannelServerVersion, }, + &cli.StringFlag{ + Name: "app-name", + Usage: "the app for which to retrieve the app default versions", + EnvVars: []string{"APP_NAME"}, + Destination: &AppName, + }, &cli.StringSliceFlag{ Name: "path-prefix", EnvVars: []string{"PATH_PREFIX"}, @@ -91,7 +98,7 @@ func run(c *cli.Context) error { sources = append(sources, config.StringSource(url)) } for index, subkey := range SubKeys.Value() { - config := config.NewConfig(ctx, subkey, &config.DurationWait{Duration: intval}, ChannelServerVersion, sources) + config := config.NewConfig(ctx, subkey, &config.DurationWait{Duration: intval}, ChannelServerVersion, AppName, sources) configs[PathPrefix.Value()[index]] = config } return server.ListenAndServe(ctx, ListenAddress, configs) diff --git a/pkg/config/config.go b/pkg/config/config.go index 2bd629c..e54cabb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -2,6 +2,7 @@ package config import ( "context" + "fmt" "net/url" "sync" "time" @@ -14,11 +15,12 @@ import ( type Config struct { sync.Mutex - url string - redirect *url.URL - gh *github.Client - channelsConfig *model.ChannelsConfig - releasesConfig *model.ReleasesConfig + url string + redirect *url.URL + gh *github.Client + channelsConfig *model.ChannelsConfig + releasesConfig *model.ReleasesConfig + appDefaultsConfig *model.AppDefaultsConfig } type Wait interface { @@ -48,21 +50,22 @@ func (s StringSource) URL() string { return string(s) } -func NewConfig(ctx context.Context, subKey string, wait Wait, channelServerVersion string, urls []Source) *Config { +func NewConfig(ctx context.Context, subKey string, wait Wait, channelServerVersion string, appName string, urls []Source) *Config { c := &Config{ - channelsConfig: &model.ChannelsConfig{}, - releasesConfig: &model.ReleasesConfig{}, + channelsConfig: &model.ChannelsConfig{}, + releasesConfig: &model.ReleasesConfig{}, + appDefaultsConfig: &model.AppDefaultsConfig{}, } - _, _ = c.loadConfig(ctx, subKey, channelServerVersion, urls...) + _, _ = c.loadConfig(ctx, subKey, channelServerVersion, appName, urls...) go func() { for wait.Wait(ctx) { - if index, err := c.loadConfig(ctx, subKey, channelServerVersion, urls...); err != nil { - logrus.Errorf("failed to reload configuration from %s: %v", urls, err) + if index, err := c.loadConfig(ctx, subKey, channelServerVersion, appName, urls...); err != nil { + logrus.Errorf("failed to reload configuration from %s: %v", urls[index].URL(), err) } else { urls = urls[:index+1] - logrus.Infof("Loaded configuration from %s in %v", urls[index], urls) + logrus.Infof("Loaded configuration from %s in %v", urls[index].URL(), urls) } } }() @@ -70,23 +73,28 @@ func NewConfig(ctx context.Context, subKey string, wait Wait, channelServerVersi return c } -func (c *Config) loadConfig(ctx context.Context, subKey string, channelServerVersion string, urls ...Source) (int, error) { +func (c *Config) loadConfig(ctx context.Context, subKey string, channelServerVersion string, appName string, urls ...Source) (int, error) { content, index, err := getURLs(ctx, urls...) if err != nil { - return index, err + return index, fmt.Errorf("failed to get content from url %s: %v", urls[index].URL(), err) } config, err := GetChannelsConfig(ctx, content, subKey) if err != nil { - return index, err + return index, fmt.Errorf("failed to get channel config: %v", err) } releases, err := GetReleasesConfig(content, channelServerVersion, subKey) if err != nil { - return index, err + return index, fmt.Errorf("failed to get release config: %v", err) } - return index, c.setConfig(ctx, channelServerVersion, config, releases) + appDefaultsConfig, err := GetAppDefaultsConfig(content, subKey, appName) + if err != nil { + return index, fmt.Errorf("failed to get app default config: %v", err) + } + + return index, c.setConfig(ctx, channelServerVersion, config, releases, appDefaultsConfig) } func (c *Config) ghClient(config *model.ChannelsConfig) (*github.Client, error) { @@ -103,7 +111,7 @@ func (c *Config) ghClient(config *model.ChannelsConfig) (*github.Client, error) return c.gh, nil } -func (c *Config) setConfig(ctx context.Context, channelServerVersion string, config *model.ChannelsConfig, releases *model.ReleasesConfig) error { +func (c *Config) setConfig(ctx context.Context, channelServerVersion string, config *model.ChannelsConfig, releases *model.ReleasesConfig, appDefaultsConfig *model.AppDefaultsConfig) error { gh, err := c.ghClient(config) if err != nil { return err @@ -132,6 +140,7 @@ func (c *Config) setConfig(ctx context.Context, channelServerVersion string, con c.channelsConfig = config c.redirect = redirect c.releasesConfig = releases + c.appDefaultsConfig = appDefaultsConfig if config.GitHub != nil { c.url = config.GitHub.APIURL } @@ -170,6 +179,12 @@ func (c *Config) ReleasesConfig() *model.ReleasesConfig { return c.releasesConfig } +func (c *Config) AppDefaultsConfig() *model.AppDefaultsConfig { + c.Lock() + defer c.Unlock() + return c.appDefaultsConfig +} + func (c *Config) Redirect(id string) (string, error) { for _, channel := range c.channelsConfig.Channels { if channel.Name == id && channel.Latest != "" { diff --git a/pkg/config/get.go b/pkg/config/get.go index 53e25c0..207bf78 100644 --- a/pkg/config/get.go +++ b/pkg/config/get.go @@ -47,6 +47,10 @@ func get(ctx context.Context, url Source) ([]byte, error) { } defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("status %v", resp.Status) + } + return ioutil.ReadAll(resp.Body) } @@ -146,3 +150,40 @@ func GetGHReleases(ctx context.Context, client *github.Client, owner, repo strin return allReleases, nil } + +func GetAppDefaultsConfig(content []byte, subKey, appName string) (*model.AppDefaultsConfig, error) { + var ( + data map[string]interface{} + allConfigs model.AppDefaultsConfig + availableConfigs model.AppDefaultsConfig + ) + + if subKey == "" { + if err := yaml.Unmarshal(content, &allConfigs); err != nil { + return nil, err + } + } else { + if err := yaml.Unmarshal(content, &data); err != nil { + return nil, err + } + subData, ok := data[subKey].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("content at key %s expected to be map[string]interface{}, found %T instead", subKey, data[subKey]) + } + if err := convert.ToObj(subData, &allConfigs); err != nil { + return nil, err + } + } + + // no app name is specified, return all AppDefaultsConfigs + if appName == "" { + return &allConfigs, nil + } + for _, appDefault := range allConfigs.AppDefaults { + if appDefault.AppName == appName { + availableConfigs.AppDefaults = append(availableConfigs.AppDefaults, appDefault) + break + } + } + return &availableConfigs, nil +} diff --git a/pkg/model/config.go b/pkg/model/config.go index fa00982..b99d3cc 100644 --- a/pkg/model/config.go +++ b/pkg/model/config.go @@ -39,3 +39,17 @@ type GitHub struct { Owner string `json:"owner,omitempty"` Repo string `json:"repo,omitempty"` } + +type AppDefaultsConfig struct { + AppDefaults []AppDefault `json:"appDefaults,omitempty"` +} + +type AppDefault struct { + AppName string `json:"appName,omitempty"` + Defaults []Default `json:"defaults,omitempty"` +} + +type Default struct { + AppVersion string `json:"appVersion,omitempty"` + DefaultVersion string `json:"defaultVersion,omitempty"` +} diff --git a/pkg/server/server.go b/pkg/server/server.go index 9699c30..50d6a88 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -15,6 +15,7 @@ import ( "github.com/rancher/channelserver/pkg/config" "github.com/rancher/channelserver/pkg/model" "github.com/rancher/channelserver/pkg/server/store" + "github.com/rancher/channelserver/pkg/server/store/appdefault" "github.com/rancher/channelserver/pkg/server/store/release" ) @@ -46,6 +47,10 @@ func NewHandler(configs map[string]*config.Config) http.Handler { schema.Store = release.New(config) schema.CollectionMethods = []string{http.MethodGet} }) + server.Schemas.MustImportAndCustomize(model.AppDefault{}, func(schema *types.APISchema) { + schema.Store = appdefault.New(config) + schema.CollectionMethods = []string{http.MethodGet} + }) prefix = strings.Trim(prefix, "/") apiroot.Register(server.Schemas, []string{prefix}) router.MatcherFunc(setType("apiRoot", prefix)).Path("/").Handler(server) diff --git a/pkg/server/store/appdefault/store.go b/pkg/server/store/appdefault/store.go new file mode 100644 index 0000000..b554256 --- /dev/null +++ b/pkg/server/store/appdefault/store.go @@ -0,0 +1,30 @@ +package appdefault + +import ( + "github.com/rancher/apiserver/pkg/store/empty" + "github.com/rancher/apiserver/pkg/types" + "github.com/rancher/channelserver/pkg/config" +) + +type Store struct { + empty.Store + config *config.Config +} + +func New(config *config.Config) *Store { + return &Store{ + config: config, + } +} + +func (c *Store) List(_ *types.APIRequest, _ *types.APISchema) (types.APIObjectList, error) { + resp := types.APIObjectList{} + for _, appDefault := range c.config.AppDefaultsConfig().AppDefaults { + resp.Objects = append(resp.Objects, types.APIObject{ + Type: "appdefault", + ID: appDefault.AppName, + Object: appDefault, + }) + } + return resp, nil +}