Skip to content

Commit

Permalink
cmd/relui: allow picking which of the private go projects to build from
Browse files Browse the repository at this point in the history 
Rather than hardcoding using golang/go-private, allow picking from that
and the "go" project on the go-internal host. This paves the way for
migrating to the new repository without needing to rollback in case of
an issue. Once we are fully migrated we'll likely go back to hardcoding
the project.

Change-Id: I380d3934279f7217964bfa5656cba642da11d66e
Reviewed-on: https://go-review.googlesource.com/c/build/+/569457
Reviewed-by: Carlos Amedee <carlos@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
  • Loading branch information
@rolandshoemaker
rolandshoemaker committed Apr 23, 2024
1 parent 7d61e25 commit 4f1971e
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 19 deletions.
13 changes: 6 additions & 7 deletions cmd/relui/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,13 +220,12 @@ func main() {
signServer := sign.NewServer()
protos.RegisterReleaseServiceServer(grpcServer, signServer)
buildTasks := &relui.BuildReleaseTasks{
GerritClient: gerritClient,
GerritProject: "go",
GerritHTTPClient: oauth2.NewClient(ctx, creds.TokenSource),
PrivateGerritClient: privateGerritClient,
PrivateGerritProject: "golang/go-private",
SignService: signServer,
GCSClient: gcsClient,
GerritClient: gerritClient,
GerritProject: "go",
GerritHTTPClient: oauth2.NewClient(ctx, creds.TokenSource),
PrivateGerritClient: privateGerritClient,
SignService: signServer,
GCSClient: gcsClient,
ScratchFS: &task.ScratchFS{
BaseURL: *scratchFilesBase,
GCS: gcsClient,
Expand Down
9 changes: 6 additions & 3 deletions internal/relui/buildrelease_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@ func testRelease(t *testing.T, prevTag string, major int, wantVersion string, ki
"js-wasm", // Builder used on 1.20 and older.
},
"Ref from the private repository to build from (optional)": "",
"Security repository to retrieve ref from (optional)": "",
})
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -368,15 +369,15 @@ func testSecurity(t *testing.T, mergeFixes bool) {

// Set up the fake merge process. Once we stop to ask for approval, commit
// the fix to the public server.
privateRepo := task.NewFakeRepo(t, "go-private")
privateRepo := task.NewFakeRepo(t, "go")
privateRepo.Commit(goFiles)
securityFix := map[string]string{"security.txt": "This file makes us secure"}
privateRepoName := "go-internal/go (new)"
privateRef := privateRepo.Commit(securityFix)
privateGerrit := task.NewFakeGerrit(t, privateRepo)
deps.buildBucket.GerritURL = privateGerrit.GerritURL()
deps.buildBucket.Projects = []string{"go-private"}
deps.buildBucket.Projects = []string{"go"}
deps.buildTasks.PrivateGerritClient = privateGerrit
deps.buildTasks.PrivateGerritProject = "go-private"

defaultApprove := deps.buildTasks.ApproveAction
deps.buildTasks.ApproveAction = func(tc *workflow.TaskContext) error {
Expand All @@ -394,6 +395,7 @@ func testSecurity(t *testing.T, mergeFixes bool) {
w, err := workflow.Start(wd, map[string]interface{}{
"Targets to skip testing (or 'all') (optional)": []string{"js-wasm"},
"Ref from the private repository to build from (optional)": privateRef,
"Security repository to retrieve ref from (optional)": privateRepoName,
})
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -438,6 +440,7 @@ func TestAdvisoryTestsFail(t *testing.T) {
w, err := workflow.Start(wd, map[string]interface{}{
"Targets to skip testing (or 'all') (optional)": []string(nil),
"Ref from the private repository to build from (optional)": "",
"Security repository to retrieve ref from (optional)": "",
})
if err != nil {
t.Fatal(err)
Expand Down
36 changes: 27 additions & 9 deletions internal/relui/workflows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ func registerBuildTestSignOnlyWorkflow(h *DefinitionHolder, version *task.Versio
versionFile := wf.Task2(wd, "Generate VERSION file", version.GenerateVersionFile, nextVersion, timestamp)
wf.Output(wd, "VERSION file", versionFile)
head := wf.Task1(wd, "Read branch head", version.ReadBranchHead, branchVal)
srcSpec := wf.Task4(wd, "Select source spec", build.getGitSource, branchVal, head, wf.Const(""), versionFile)
srcSpec := wf.Task5(wd, "Select source spec", build.getGitSource, branchVal, head, wf.Const(""), wf.Const(""), versionFile)
source, artifacts, mods := build.addBuildTasks(wd, major, kind, nextVersion, timestamp, srcSpec)
wf.Output(wd, "Source", source)
wf.Output(wd, "Artifacts", artifacts)
Expand Down Expand Up @@ -434,6 +434,11 @@ func now(_ context.Context) (time.Time, error) {
return time.Now().UTC().Round(time.Second), nil
}

var securityProjectNameToProject = map[string]string{
"go-internal/go (new)": "go",
"go-internal/golang/go-private (old)": "golang/go-private",
}

func addSingleReleaseWorkflow(
build *BuildReleaseTasks, milestone *task.MilestoneTasks, version *task.VersionTasks,
wd *wf.Definition, major int, kind task.ReleaseKind, coordinators wf.Value[[]string],
Expand All @@ -454,9 +459,23 @@ func addSingleReleaseWorkflow(
milestones := wf.Task2(wd, "Pick milestones", milestone.FetchMilestones, nextVersion, kindVal)
checked := wf.Action3(wd, "Check blocking issues", milestone.CheckBlockers, milestones, nextVersion, kindVal)

securityProjectName := wf.Param(wd, wf.ParamDef[string]{
Name: "Security repository to retrieve ref from (optional)",
ParamType: workflow.ParamType[string]{
HTMLElement: "select",
HTMLSelectOptions: []string{
"go-internal/go (new)",
"go-internal/golang/go-private (old)",
},
},
Doc: `"go-internal/golang/go-private" is the old internal gerrit repository, "go-internal/go" is the new repository.`,
})
securityProject := wf.Task1(wd, "Convert security project name", func(ctx *wf.TaskContext, projectName string) (string, error) {
return securityProjectNameToProject[projectName], nil
}, securityProjectName)
securityRef := wf.Param(wd, wf.ParamDef[string]{Name: "Ref from the private repository to build from (optional)"})
securityCommit := wf.Task1(wd, "Read security ref", build.readSecurityRef, securityRef)
srcSpec := wf.Task4(wd, "Select source spec", build.getGitSource, branchVal, startingHead, securityCommit, versionFile, wf.After(checked))
securityCommit := wf.Task2(wd, "Read security ref", build.readSecurityRef, securityProject, securityRef)
srcSpec := wf.Task5(wd, "Select source spec", build.getGitSource, branchVal, startingHead, securityProject, securityCommit, versionFile, wf.After(checked))

// Build, test, and sign release.
source, signedAndTestedArtifacts, modules := build.addBuildTasks(wd, major, kind, nextVersion, timestamp, srcSpec)
Expand Down Expand Up @@ -610,7 +629,6 @@ type BuildReleaseTasks struct {
GerritProject string
GerritHTTPClient *http.Client // GerritHTTPClient is an HTTP client that authenticates to Gerrit instances. (Both public and private.)
PrivateGerritClient task.GerritClient
PrivateGerritProject string
GCSClient *storage.Client
ScratchFS *task.ScratchFS
SignedURL string // SignedURL is a gs:// or file:// URL, no trailing slash.
Expand All @@ -629,24 +647,24 @@ type BuildReleaseTasks struct {

var commitRE = regexp.MustCompile(`[a-f0-9]{40}`)

func (b *BuildReleaseTasks) readSecurityRef(ctx *wf.TaskContext, ref string) (string, error) {
func (b *BuildReleaseTasks) readSecurityRef(ctx *wf.TaskContext, project, ref string) (string, error) {
if ref == "" {
return "", nil
}
if commitRE.MatchString(ref) {
return ref, nil
}
commit, err := b.PrivateGerritClient.ReadBranchHead(ctx, b.PrivateGerritProject, ref)
commit, err := b.PrivateGerritClient.ReadBranchHead(ctx, project, ref)
if err != nil {
return "", fmt.Errorf("%q doesn't appear to be a commit hash, but resolving it as a branch failed: %v", ref, err)
}
return commit, nil
}

func (b *BuildReleaseTasks) getGitSource(ctx *wf.TaskContext, branch, commit, securityCommit, versionFile string) (sourceSpec, error) {
func (b *BuildReleaseTasks) getGitSource(ctx *wf.TaskContext, branch, commit, securityProject, securityCommit, versionFile string) (sourceSpec, error) {
client, project, rev := b.GerritClient, b.GerritProject, commit
if securityCommit != "" {
client, project, rev = b.PrivateGerritClient, b.PrivateGerritProject, securityCommit
client, project, rev = b.PrivateGerritClient, securityProject, securityCommit
}
return sourceSpec{
GitilesURL: client.GitilesURL(),
Expand Down Expand Up @@ -718,7 +736,7 @@ func (b *BuildReleaseTasks) checkSourceMatch(ctx *wf.TaskContext, branch, versio
if err != nil {
return "", err
}
spec, err := b.getGitSource(ctx, branch, head, "", versionFile)
spec, err := b.getGitSource(ctx, branch, head, "", "", versionFile)
if err != nil {
return "", err
}
Expand Down

0 comments on commit 4f1971e

Please sign in to comment.