Merge branch 'master' into document_quality_bar

andrewpollock · May 17, 2024 · d75f4aa · d75f4aa
2 parents e473285 + 2e1117e
commit d75f4aa
Show file tree

Hide file tree

Showing 31 changed files with 1,059 additions and 765 deletions.
diff --git a/.github/workflows/issue-signposting.yml b/.github/workflows/issue-signposting.yml
@@ -0,0 +1,22 @@
+name: Add comments to specifically labeled issues
+on:
+  issues:
+    types:
+      - labeled
+jobs:
+  add-comment:
+    if: github.event.label.name == 'data quality'
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+      - name: Add comment signposting to FAQ
+        run: gh issue comment "$NUMBER" --body "$BODY"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          NUMBER: ${{ github.event.issue.number }}
+          BODY: >
+            :sparkles: Thank you for your interest in OSV.dev's data quality! :sparkles:
+            **Please review our [FAQ entry](https://google.github.io/osv.dev/faq/#ive-found-something-wrong-with-the-data) on how to most efficiently have this addressed.**
+
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/deployment/clouddeploy/gke-workers/base/importer-deleter.yaml b/deployment/clouddeploy/gke-workers/base/importer-deleter.yaml
@@ -0,0 +1,32 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: importer-deleter
+spec:
+  schedule: "* */3 * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: importer-deleter
+            image: importer
+            imagePullPolicy: Always
+            volumeMounts:
+              - mountPath: "/work"
+                name: "ssd"
+            securityContext:
+              privileged: true
+            resources:
+              requests:
+                cpu: 1
+                memory: "10G"
+              limits:
+                cpu: 1
+                memory: "11G"
+          restartPolicy: OnFailure
+          volumes:
+            - name: "ssd"
+              hostPath:
+                path: "/mnt/disks/ssd0"
diff --git a/deployment/clouddeploy/gke-workers/base/kustomization.yaml b/deployment/clouddeploy/gke-workers/base/kustomization.yaml
@@ -2,6 +2,7 @@ resources:
 - workers.yaml
 - scaler.yaml
 - importer.yaml
+- importer-deleter.yaml
 - exporter.yaml
 - alpine-cve-convert.yaml
 - combine-to-osv.yaml

diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/importer-deleter.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/importer-deleter.yaml
@@ -3,8 +3,6 @@ kind: CronJob
 metadata:
   name: importer-deleter
 spec:
-  schedule: "* */6 * * *"
-  concurrencyPolicy: Forbid
   jobTemplate:
     spec:
       template:
@@ -19,21 +17,3 @@ spec:
               - --delete
               - --delete_threshold_pct=20
               - --public_log_bucket=osv-test-public-import-logs
-            imagePullPolicy: Always
-            volumeMounts:
-              - mountPath: "/work"
-                name: "ssd"
-            securityContext:
-              privileged: true
-            resources:
-              requests:
-                cpu: 1
-                memory: "10G"
-              limits:
-                cpu: 1
-                memory: "11G"
-          restartPolicy: OnFailure
-          volumes:
-            - name: "ssd"
-              hostPath:
-                path: "/mnt/disks/ssd0"
diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml
@@ -1,11 +1,11 @@
 resources:
 - ../../base
 - debian-cve-convert.yaml
-- importer-deleter.yaml
 patches:
 - path: workers.yaml
 - path: scaler.yaml
 - path: importer.yaml
+- path: importer-deleter.yaml
 - path: exporter.yaml
 - path: alpine-cve-convert.yaml
 - path: combine-to-osv.yaml

diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb/importer-deleter.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb/importer-deleter.yaml
@@ -0,0 +1,19 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: importer-deleter
+spec:
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: importer-deleter
+            env:
+            - name: GOOGLE_CLOUD_PROJECT
+              value: oss-vdb
+            image: importer
+            args:
+              - --delete
+              - --delete_threshold_pct=20
+              - --public_log_bucket=osv-public-import-logs
diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb/kustomization.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb/kustomization.yaml
@@ -5,6 +5,7 @@ resources:
 patches:
 - path: workers.yaml
 - path: importer.yaml
+- path: importer-deleter.yaml
 - path: exporter.yaml
 - path: alpine-cve-convert.yaml
 - path: combine-to-osv.yaml

diff --git a/deployment/terraform/environments/oss-vdb-test/.terraform.lock.hcl b/deployment/terraform/environments/oss-vdb-test/.terraform.lock.hcl
diff --git a/deployment/terraform/environments/oss-vdb-test/main.tf b/deployment/terraform/environments/oss-vdb-test/main.tf
@@ -13,8 +13,14 @@ module "osv_test" {
   affected_commits_backups_bucket                = "osv-test-affected-commits"
   affected_commits_backups_bucket_retention_days = 2
 
-  api_url     = "api.test.osv.dev"
-  esp_version = "2.47.0"
+  website_domain = "test.osv.dev"
+  api_url        = "api.test.osv.dev"
+  esp_version    = "2.47.0"
+}
+
+output "website_dns_records" {
+  description = "DNS records that need to be created for the osv.dev website"
+  value       = module.osv_test.website_dns_records
 }
 
 
@@ -27,11 +33,11 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 5.18.0"
+      version = "~> 5.27.0"
     }
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = "~> 5.18.0"
+      version = "~> 5.27.0"
     }
     external = {
       source  = "hashicorp/external"

diff --git a/deployment/terraform/environments/oss-vdb/.terraform.lock.hcl b/deployment/terraform/environments/oss-vdb/.terraform.lock.hcl
diff --git a/deployment/terraform/environments/oss-vdb/main.tf b/deployment/terraform/environments/oss-vdb/main.tf
@@ -13,10 +13,15 @@ module "osv" {
   affected_commits_backups_bucket                = "osv-affected-commits"
   affected_commits_backups_bucket_retention_days = 3
 
-  api_url     = "api.osv.dev"
-  esp_version = "2.47.0"
+  website_domain = "osv.dev"
+  api_url        = "api.osv.dev"
+  esp_version    = "2.47.0"
 }
 
+output "website_dns_records" {
+  description = "DNS records that need to be created for the osv.dev website"
+  value       = module.osv.website_dns_records
+}
 
 terraform {
   backend "gcs" {
@@ -26,11 +31,11 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 5.18.0"
+      version = "~> 5.27.0"
     }
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = "~> 5.18.0"
+      version = "~> 5.27.0"
     }
     external = {
       source  = "hashicorp/external"

diff --git a/deployment/terraform/modules/osv/variables.tf b/deployment/terraform/modules/osv/variables.tf
@@ -57,3 +57,8 @@ variable "esp_version" {
   type        = string
   description = "ESP version to use for OSV API frontend image."
 }
+
+variable "website_domain" {
+  type        = string
+  description = "Domain to serve the OSV website on. Domain ownership and DNS settings must be manually configured."
+}