Add cpature_corrupted_outputs (#1179)

buildkite/bazelci.py

@@ -966,23 +966,31 @@ def bazelisk_flags():
     return ["--migrate"] if use_bazelisk_migrate() else []
 
 
-def calculate_flags(task_config, task_config_key, json_profile_key, tmpdir, test_env_vars):
+def calculate_flags(task_config, task_config_key, action_key, tmpdir, test_env_vars):
     include_json_profile = task_config.get("include_json_profile", [])
+    capture_corrupted_outputs = task_config.get("capture_corrupted_outputs", [])
 
     json_profile_flags = []
     json_profile_out = None
-    if json_profile_key in include_json_profile:
-        json_profile_out = os.path.join(tmpdir, "{}.profile.gz".format(json_profile_key))
+    if action_key in include_json_profile:
+        json_profile_out = os.path.join(tmpdir, "{}.profile.gz".format(action_key))
         json_profile_flags = get_json_profile_flags(json_profile_out)
+
+    capture_corrupted_outputs_flags = []
+    capture_corrupted_outputs_dir = None
+    if action_key in capture_corrupted_outputs:
+        capture_corrupted_outputs_dir = os.path.join(tmpdir, "{}_corrupted_outputs".format(action_key))
+        capture_corrupted_outputs_flags = ["--experimental_remote_capture_corrupted_outputs={}".format(capture_corrupted_outputs_dir)]
 
     flags = task_config.get(task_config_key) or []
     flags += json_profile_flags
+    flags += capture_corrupted_outputs_flags
     # We have to add --test_env flags to `build`, too, otherwise Bazel
     # discards its analysis cache between `build` and `test`.
     if test_env_vars:
         flags += ["--test_env={}".format(v) for v in test_env_vars]
 
-    return flags, json_profile_out
+    return flags, json_profile_out, capture_corrupted_outputs_dir
 
 
 def execute_commands(
@@ -1111,7 +1119,7 @@ def execute_commands(
         )
 
         if build_targets:
-            build_flags, json_profile_out_build = calculate_flags(
+            build_flags, json_profile_out_build, capture_corrupted_outputs_dir_build = calculate_flags(
                 task_config, "build_flags", "build", tmpdir, test_env_vars
             )
             try:
@@ -1129,9 +1137,11 @@ def execute_commands(
             finally:
                 if json_profile_out_build:
                     upload_json_profile(json_profile_out_build, tmpdir)
+                if capture_corrupted_outputs_dir_build:
+                    upload_corrupted_outputs(capture_corrupted_outputs_dir_build, tmpdir)
 
         if test_targets:
-            test_flags, json_profile_out_test = calculate_flags(
+            test_flags, json_profile_out_test, capture_corrupted_outputs_dir_test = calculate_flags(
                 task_config, "test_flags", "test", tmpdir, test_env_vars
             )
             if not is_windows():
@@ -1167,12 +1177,14 @@ def execute_commands(
                 finally:
                     if json_profile_out_test:
                         upload_json_profile(json_profile_out_test, tmpdir)
+                    if capture_corrupted_outputs_dir_test:
+                        upload_corrupted_outputs(capture_corrupted_outputs_dir_test, tmpdir)
             finally:
                 stop_request.set()
                 upload_thread.join()
 
         if index_targets:
-            index_flags, json_profile_out_index = calculate_flags(
+            index_flags, json_profile_out_index, capture_corrupted_outputs_dir_index = calculate_flags(
                 task_config, "index_flags", "index", tmpdir, test_env_vars
             )
             index_upload_policy = task_config.get("index_upload_policy", "IfBuildSuccess")
@@ -1208,6 +1220,8 @@ def execute_commands(
             finally:
                 if json_profile_out_index:
                     upload_json_profile(json_profile_out_index, tmpdir)
+                if capture_corrupted_outputs_dir_index:
+                    upload_corrupted_outputs(capture_corrupted_outputs_dir_index, tmpdir)
 
     finally:
         terminate_background_process(sc_process)
@@ -2011,6 +2025,11 @@ def upload_json_profile(json_profile_path, tmpdir):
     print_collapsed_group(":gcloud: Uploading JSON Profile")
     execute_command(["buildkite-agent", "artifact", "upload", json_profile_path], cwd=tmpdir)
 
+def upload_corrupted_outputs(capture_corrupted_outputs_dir, tmpdir):
+    if not os.path.exists(capture_corrupted_outputs_dir):
+        return
+    print_collapsed_group(":gcloud: Uploading corrupted outputs")
+    execute_command(["buildkite-agent", "artifact", "upload", "{}/**/*".format(capture_corrupted_outputs_dir)], cwd=tmpdir)
 
 def rename_test_logs_for_upload(test_logs, tmpdir):
     # Rename the test.log files to the target that created them