fix: tekton artifact list (#80)

# Why:
- pipeline url typo (we can make it configurable later)
- fix build report flash old artifacts while processing new event

# How:
- fix typo
- avoid flash BuildReport.Images every time while processing each
pipeline event

---------

Signed-off-by: lijie <lijie@pingcap.com>

tibuild/api/api.go

@@ -103,7 +103,7 @@ func routeRestAPI(router *gin.Engine, cfg *configs.ConfigYaml) {
 		panic(err)
 	}
 	devBuildGroup := apiGroup.Group("/devbuilds")
-	devBuildServer := controllers.NewDevBuildServer(jenkins, database.DBConn.DB, cfg.CloudEvent.Endpoint, cfg.Github.Token)
+	devBuildServer := controllers.NewDevBuildServer(jenkins, database.DBConn.DB, cfg)
 	devBuildHandler := controllers.NewDevBuildHandler(devBuildServer, cfg.RestApiSecret)
 	{
 		devBuildGroup.POST("", devBuildHandler.Create)

tibuild/commons/configs/config.go

@@ -35,6 +35,9 @@ type ConfigYaml struct {
 	CloudEvent    struct {
 		Endpoint string
 	}
+
+	TektonViewURL    string
+	OciFileserverURL string
 }
 
 type RestApiSecret struct {

tibuild/commons/configs/config_test.go

@@ -10,4 +10,5 @@ func TestLoadConfig(t *testing.T) {
 	LoadConfig("../../configs/config_example.yaml")
 	cfg := Config.RestApiSecret
 	assert.NotEmpty(t, cfg.TiBuildToken)
+	assert.NotEmpty(t, Config.OciFileserverURL)
 }

tibuild/configs/config_example.yaml

@@ -20,3 +20,10 @@ github:
 restapisecret:
   admintoken: "1"
   tibuildtoken: "2"
+
+cloudevent:
+  endpoint: "http://localhost:8000"
+
+tektonviewurl: "https://do.pingcap.net/tekton/#/namespaces/ee-cd/pipelineruns"
+
+ocifileserverurl: "https://internal.do.pingcap.net:30443/dl/oci-file"

tibuild/docs/docs.go

@@ -553,7 +553,7 @@ const docTemplate = `{
                 }
             }
         },
-        "service.OrasArtifact": {
+        "service.OciArtifact": {
             "type": "object",
             "properties": {
                 "files": {
@@ -687,7 +687,7 @@ const docTemplate = `{
                 "orasArtifacts": {
                     "type": "array",
                     "items": {
-                        "$ref": "#/definitions/service.OrasArtifact"
+                        "$ref": "#/definitions/service.OciArtifact"
                     }
                 },
                 "pipelineEndAt": {

tibuild/docs/swagger.json

@@ -541,7 +541,7 @@
                 }
             }
         },
-        "service.OrasArtifact": {
+        "service.OciArtifact": {
             "type": "object",
             "properties": {
                 "files": {
@@ -675,7 +675,7 @@
                 "orasArtifacts": {
                     "type": "array",
                     "items": {
-                        "$ref": "#/definitions/service.OrasArtifact"
+                        "$ref": "#/definitions/service.OciArtifact"
                     }
                 },
                 "pipelineEndAt": {

tibuild/docs/swagger.yaml

@@ -154,7 +154,7 @@ definitions:
       target:
         type: string
     type: object
-  service.OrasArtifact:
+  service.OciArtifact:
     properties:
       files:
         items:
@@ -254,7 +254,7 @@ definitions:
         type: string
       orasArtifacts:
         items:
-          $ref: '#/definitions/service.OrasArtifact'
+          $ref: '#/definitions/service.OciArtifact'
         type: array
       pipelineEndAt:
         type: string

tibuild/pkg/rest/controller/dev_build_handler.go

@@ -27,14 +27,16 @@ func NewDevBuildHandler(svc service.DevBuildService, auth configs.RestApiSecret)
 	}
 }
 
-func NewDevBuildServer(jenkins service.Jenkins, db *gorm.DB, ce_endpoint string, gh_token string) service.DevBuildService {
+func NewDevBuildServer(jenkins service.Jenkins, db *gorm.DB, cfg *configs.ConfigYaml) service.DevBuildService {
 	db.AutoMigrate(&service.DevBuild{})
 	return &service.DevbuildServer{
-		Repo:     repo.DevBuildRepo{Db: db},
-		Jenkins:  jenkins,
-		Now:      time.Now,
-		Tekton:   service.NewCEClient(ce_endpoint),
-		GHClient: service.NewGHClient(gh_token),
+		Repo:             repo.DevBuildRepo{Db: db},
+		Jenkins:          jenkins,
+		Now:              time.Now,
+		Tekton:           service.NewCEClient(cfg.CloudEvent.Endpoint),
+		GHClient:         service.NewGHClient(cfg.Github.Token),
+		TektonViewURL:    cfg.TektonViewURL,
+		OciFileserverURL: cfg.OciFileserverURL,
 	}
 }
 

tibuild/pkg/rest/service/dev_build_service.go

@@ -12,11 +12,13 @@ import (
 )
 
 type DevbuildServer struct {
-	Repo     DevBuildRepository
-	Jenkins  Jenkins
-	Tekton   BuildTrigger
-	Now      func() time.Time
-	GHClient GHClient
+	Repo             DevBuildRepository
+	Jenkins          Jenkins
+	Tekton           BuildTrigger
+	Now              func() time.Time
+	GHClient         GHClient
+	TektonViewURL    string
+	OciFileserverURL string
 }
 
 const jobname = "devbuild"
@@ -338,13 +340,13 @@ func (s DevbuildServer) inflate(entity *DevBuild) {
 	if entity.Status.BuildReport != nil {
 		for i, bin := range entity.Status.BuildReport.Binaries {
 			if bin.URL == "" && bin.OrasFile != nil {
-				entity.Status.BuildReport.Binaries[i].URL = oras_to_file_url(*bin.OrasFile)
+				entity.Status.BuildReport.Binaries[i].URL = s.oras_to_file_url(*bin.OrasFile)
 			}
 		}
 	}
 	if tek := entity.Status.TektonStatus; tek != nil {
 		for i, p := range tek.Pipelines {
-			tek.Pipelines[i].URL = fmt.Sprintf("%s/%s", tektonURL, p.Name)
+			tek.Pipelines[i].URL = fmt.Sprintf("%s/%s", s.TektonViewURL, p.Name)
 		}
 	}
 }
@@ -370,80 +372,99 @@ func (s DevbuildServer) MergeTektonStatus(ctx context.Context, id int, pipeline
 	} else {
 		status.Pipelines = append(status.Pipelines, pipeline)
 	}
-	compute_tekton_status(status)
+	computeTektonStatus(status)
 	if obj.Spec.PipelineEngine == TektonEngine {
 		obj.Status.Status = obj.Status.TektonStatus.Status
 		obj.Status.BuildReport = obj.Status.TektonStatus.BuildReport
 	}
 	return s.Update(ctx, id, *obj, options)
 }
 
-func compute_tekton_status(status *TektonStatus) {
+func computeTektonStatus(status *TektonStatus) {
+	status.BuildReport = &BuildReport{}
+	collectTektonArtifacts(status.Pipelines, status.BuildReport)
+	status.PipelineStartAt = getTektonStartAt(status.Pipelines)
+	status.Status = computeTektonPhase(status.Pipelines)
+	if status.Status.IsCompleted() {
+		status.PipelineEndAt = getLatestEndAt(status.Pipelines)
+	}
+}
+
+func collectTektonArtifacts(pipelines []TektonPipeline, report *BuildReport) {
+	for _, pipeline := range pipelines {
+		report.GitHash = pipeline.GitHash
+		for _, files := range pipeline.OciArtifacts {
+			report.Binaries = append(report.Binaries, oras_to_files(pipeline.Platform, files)...)
+		}
+		for _, image := range pipeline.Images {
+			img := ImageArtifact{Platform: pipeline.Platform, URL: image.URL}
+			report.Images = append(report.Images, img)
+		}
+	}
+}
+
+func getTektonStartAt(pipelines []TektonPipeline) *time.Time {
+	var startAt *time.Time = nil
+	for _, pipeline := range pipelines {
+		if pipeline.PipelineStartAt != nil {
+			if startAt == nil {
+				startAt = pipeline.PipelineStartAt
+			} else if pipeline.PipelineStartAt.Before(*startAt) {
+				startAt = pipeline.PipelineStartAt
+			}
+		}
+	}
+	return startAt
+}
+
+func computeTektonPhase(pipelines []TektonPipeline) BuildStatus {
 	phase := BuildStatusPending
 	var success_platforms = map[Platform]struct{}{}
 	var failure_platforms = map[Platform]struct{}{}
 	var triggered_platforms = map[Platform]struct{}{}
-	var latest_endat *time.Time
-	for _, pipeline := range status.Pipelines {
+	for _, pipeline := range pipelines {
 		switch pipeline.Status {
 		case BuildStatusSuccess:
 			success_platforms[pipeline.Platform] = struct{}{}
 		case BuildStatusFailure:
 			failure_platforms[pipeline.Platform] = struct{}{}
 		}
 		triggered_platforms[pipeline.Platform] = struct{}{}
-		if status.BuildReport == nil {
-			status.BuildReport = &BuildReport{}
-		} else {
-			status.BuildReport.Images = nil
-			status.BuildReport.Binaries = nil
-		}
-		status.BuildReport.GitHash = pipeline.GitHash
-		for _, files := range pipeline.OrasArtifacts {
-			status.BuildReport.Binaries = append(status.BuildReport.Binaries, oras_to_files(pipeline.Platform, files)...)
-		}
-		for _, image := range pipeline.Images {
-			img := ImageArtifact{Platform: pipeline.Platform, URL: image.URL}
-			status.BuildReport.Images = append(status.BuildReport.Images, img)
-		}
-		if pipeline.PipelineStartAt != nil {
-			if status.PipelineStartAt == nil {
-				status.PipelineStartAt = pipeline.PipelineStartAt
-			} else if pipeline.PipelineStartAt.Before(*status.PipelineStartAt) {
-				status.PipelineStartAt = pipeline.PipelineStartAt
-			}
-		}
-		if pipeline.PipelineEndAt != nil {
-			if latest_endat == nil {
-				latest_endat = pipeline.PipelineEndAt
-			} else if latest_endat.Before(*pipeline.PipelineEndAt) {
-				latest_endat = pipeline.PipelineEndAt
-			}
-		}
 	}
 	if len(success_platforms) == len(triggered_platforms) {
 		phase = BuildStatusSuccess
 	} else if len(failure_platforms) != 0 {
 		phase = BuildStatusFailure
-	} else if len(status.Pipelines) != 0 {
+	} else if len(pipelines) != 0 {
 		phase = BuildStatusProcessing
 	}
-	status.Status = phase
-	if status.Status.IsCompleted() {
-		status.PipelineEndAt = latest_endat
+	return phase
+}
+
+func getLatestEndAt(pipelines []TektonPipeline) *time.Time {
+	var latest_endat *time.Time
+	for _, pipeline := range pipelines {
+		if pipeline.PipelineEndAt != nil {
+			if latest_endat == nil {
+				latest_endat = pipeline.PipelineEndAt
+			} else if latest_endat.Before(*pipeline.PipelineEndAt) {
+				latest_endat = pipeline.PipelineEndAt
+			}
+		}
 	}
+	return latest_endat
 }
 
-func oras_to_files(platform Platform, oras OrasArtifact) []BinArtifact {
+func oras_to_files(platform Platform, oras OciArtifact) []BinArtifact {
 	var rt []BinArtifact
 	for _, file := range oras.Files {
 		rt = append(rt, BinArtifact{Platform: platform, OrasFile: &OrasFile{Repo: oras.Repo, Tag: oras.Tag, File: file}})
 	}
 	return rt
 }
 
-func oras_to_file_url(oras OrasFile) string {
-	return fmt.Sprintf("%s/oci-file/%s?tag=%s&file=%s", oras_fileserver_url, oras.Repo, oras.Tag, oras.File)
+func (s DevbuildServer) oras_to_file_url(oras OrasFile) string {
+	return fmt.Sprintf("%s/%s?tag=%s&file=%s", s.OciFileserverURL, oras.Repo, oras.Tag, oras.File)
 }
 
 type DevBuildRepository interface {
@@ -459,6 +480,3 @@ var versionValidator *regexp.Regexp = regexp.MustCompile(`^v(\d+\.\d+)(\.\d+).*$
 var hotfixVersionValidator *regexp.Regexp = regexp.MustCompile(`^v(\d+\.\d+)\.\d+-\d{8,}.*$`)
 var gitRefValidator *regexp.Regexp = regexp.MustCompile(`^((v\d.*)|(pull/\d+)|([0-9a-fA-F]{40})|(release-.*)|master|main|(tag/.+)|(branch/.+))$`)
 var githubRepoValidator *regexp.Regexp = regexp.MustCompile(`^([\w_-]+/[\w_-]+)$`)
-
-const tektonURL = "https://do.pingcap.net/tekton/#/namespaces/ee-cd/pipelineruns/"
-const oras_fileserver_url = "https://internal.do.pingcap.net:30443/dl"

tibuild/pkg/rest/service/dev_build_service_test.go

@@ -270,9 +270,11 @@ func TestDevBuildGet(t *testing.T) {
 	mockedRepo := mockRepo{}
 	mockedJenkins := mockJenkins{}
 	server := DevbuildServer{
-		Repo:    &mockedRepo,
-		Jenkins: &mockedJenkins,
-		Now:     time.Now,
+		Repo:             &mockedRepo,
+		Jenkins:          &mockedJenkins,
+		Now:              time.Now,
+		TektonViewURL:    "http://tekton.net",
+		OciFileserverURL: "http://orasdownload.net",
 	}
 
 	t.Run("ok", func(t *testing.T) {
@@ -289,15 +291,15 @@ func TestDevBuildGet(t *testing.T) {
 			Status: DevBuildStatus{PipelineBuildID: 4, BuildReport: &BuildReport{Binaries: []BinArtifact{{OrasFile: &OrasFile{Repo: "repo", Tag: "tag", File: "file"}}}}}}
 		entity, err := server.Get(context.TODO(), 1, DevBuildGetOption{})
 		require.NoError(t, err)
-		require.Equal(t, "https://internal.do.pingcap.net:30443/dl/oci-file/repo?tag=tag&file=file", entity.Status.BuildReport.Binaries[0].URL)
+		require.Equal(t, "http://orasdownload.net/repo?tag=tag&file=file", entity.Status.BuildReport.Binaries[0].URL)
 	})
 	t.Run("render tekton pipeline", func(t *testing.T) {
 		mockedRepo.saved = DevBuild{ID: 1,
 			Spec:   DevBuildSpec{Product: ProductTidb, Version: "v6.1.2", Edition: EnterpriseEdition, GitRef: "pull/23", PluginGitRef: "master"},
 			Status: DevBuildStatus{PipelineBuildID: 4, TektonStatus: &TektonStatus{Pipelines: []TektonPipeline{{Name: "p1"}}}}}
 		entity, err := server.Get(context.TODO(), 1, DevBuildGetOption{})
 		require.NoError(t, err)
-		require.Equal(t, tektonURL+"/p1", entity.Status.TektonStatus.Pipelines[0].URL)
+		require.Equal(t, "http://tekton.net/p1", entity.Status.TektonStatus.Pipelines[0].URL)
 	})
 	t.Run("sync", func(t *testing.T) {
 		mockedRepo.saved = DevBuild{ID: 1,
@@ -328,7 +330,7 @@ func TestMergeTektonStatus(t *testing.T) {
 				{Name: "p1", Platform: LinuxAmd64, Status: BuildStatusSuccess},
 				{Name: "p2", Platform: LinuxArm64, Status: BuildStatusProcessing},
 			}}}}
-		pipelinerun := TektonPipeline{Name: "p2", Platform: LinuxArm64, Status: BuildStatusSuccess, OrasArtifacts: []OrasArtifact{{Repo: "repo", Tag: "tag", Files: []string{"file1.tar.gz", "file2.tar.gz"}}}}
+		pipelinerun := TektonPipeline{Name: "p2", Platform: LinuxArm64, Status: BuildStatusSuccess, OciArtifacts: []OciArtifact{{Repo: "repo", Tag: "tag", Files: []string{"file1.tar.gz", "file2.tar.gz"}}}}
 		entity, err := server.MergeTektonStatus(context.TODO(), 1, pipelinerun, DevBuildSaveOption{})
 		require.NoError(t, err)
 		require.Equal(t, BuildStatusSuccess, entity.Status.Status)
@@ -365,49 +367,52 @@ func TestTektonStatusMerge(t *testing.T) {
 			Pipelines: []TektonPipeline{
 				{Name: "pipelinerun1", Status: BuildStatusSuccess, Platform: LinuxAmd64,
 					PipelineStartAt: &starttime,
-					OrasArtifacts:   []OrasArtifact{{Repo: "harbor.net/org/repo", Files: []string{"a.tar.gz", "b.tar.gz"}}},
+					OciArtifacts:    []OciArtifact{{Repo: "harbor.net/org/repo", Tag: "master", Files: []string{"a.tar.gz", "b.tar.gz"}}},
 					Images:          []ImageArtifact{{URL: "harbor.net/org/image:tag1"}}},
 				{Name: "pipelinerun2", Status: BuildStatusSuccess, Platform: LinuxArm64,
 					PipelineStartAt: &starttime,
 					PipelineEndAt:   &endtime,
-					OrasArtifacts:   []OrasArtifact{{Repo: "harbor.net/org/repo", Files: []string{"c.tar.gz", "d.tar.gz"}}}},
+					Images:          []ImageArtifact{{URL: "harbor.net/org/image:tag2"}},
+					OciArtifacts:    []OciArtifact{{Repo: "harbor.net/org/repo", Tag: "master", Files: []string{"c.tar.gz", "d.tar.gz"}}}},
 			},
 		}
-		compute_tekton_status(status)
+		computeTektonStatus(status)
 		require.Equal(t, BuildStatusSuccess, status.Status)
 		require.Equal(t, endtime.Sub(starttime), status.PipelineEndAt.Sub(*status.PipelineStartAt))
+		require.Equal(t, 2, len(status.BuildReport.Images))
+		require.Equal(t, 4, len(status.BuildReport.Binaries))
 	})
 
 	t.Run("processing", func(t *testing.T) {
 		status := &TektonStatus{
 			Pipelines: []TektonPipeline{
 				{Name: "pipelinerun1", Status: BuildStatusSuccess, Platform: LinuxAmd64,
 					PipelineStartAt: &starttime,
-					OrasArtifacts:   []OrasArtifact{{Repo: "harbor.net/org/repo", Files: []string{"a.tar.gz", "b.tar.gz"}}},
+					OciArtifacts:    []OciArtifact{{Repo: "harbor.net/org/repo", Files: []string{"a.tar.gz", "b.tar.gz"}}},
 					Images:          []ImageArtifact{{URL: "harbor.net/org/image:tag1"}}},
 				{Name: "pipelinerun2", Status: BuildStatusProcessing, Platform: LinuxArm64,
 					PipelineStartAt: &starttime,
 					PipelineEndAt:   &endtime,
-					OrasArtifacts:   []OrasArtifact{{Repo: "harbor.net/org/repo", Files: []string{"c.tar.gz", "d.tar.gz"}}}},
+					OciArtifacts:    []OciArtifact{{Repo: "harbor.net/org/repo", Files: []string{"c.tar.gz", "d.tar.gz"}}}},
 			},
 		}
-		compute_tekton_status(status)
+		computeTektonStatus(status)
 		require.Equal(t, BuildStatusProcessing, status.Status)
 	})
 	t.Run("processing", func(t *testing.T) {
 		status := &TektonStatus{
 			Pipelines: []TektonPipeline{
 				{Name: "pipelinerun1", Status: BuildStatusSuccess, Platform: LinuxAmd64,
 					PipelineStartAt: &starttime,
-					OrasArtifacts:   []OrasArtifact{{Repo: "harbor.net/org/repo", Tag: "latest", Files: []string{"a.tar.gz", "b.tar.gz"}}},
+					OciArtifacts:    []OciArtifact{{Repo: "harbor.net/org/repo", Tag: "latest", Files: []string{"a.tar.gz", "b.tar.gz"}}},
 					Images:          []ImageArtifact{{URL: "harbor.net/org/image:tag1"}}},
 				{Name: "pipelinerun2", Status: BuildStatusFailure, Platform: LinuxArm64,
 					PipelineStartAt: &starttime,
 					PipelineEndAt:   &endtime,
-					OrasArtifacts:   []OrasArtifact{{Repo: "harbor.net/org/repo", Files: []string{"c.tar.gz", "d.tar.gz"}}}},
+					OciArtifacts:    []OciArtifact{{Repo: "harbor.net/org/repo", Files: []string{"c.tar.gz", "d.tar.gz"}}}},
 			},
 		}
-		compute_tekton_status(status)
+		computeTektonStatus(status)
 		require.Equal(t, BuildStatusFailure, status.Status)
 	})