forked from bazelbuild/bazel
-
Notifications
You must be signed in to change notification settings - Fork 0
/
StarlarkAction.java
466 lines (432 loc) · 18.3 KB
/
StarlarkAction.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
// Copyright 2019 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.analysis.actions;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.devtools.build.lib.actions.Action;
import com.google.devtools.build.lib.actions.ActionCacheAwareAction;
import com.google.devtools.build.lib.actions.ActionEnvironment;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionExecutionException;
import com.google.devtools.build.lib.actions.ActionOwner;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.CommandLineExpansionException;
import com.google.devtools.build.lib.actions.CommandLines;
import com.google.devtools.build.lib.actions.CommandLines.CommandLineLimits;
import com.google.devtools.build.lib.actions.EnvironmentalExecException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.ExecutionRequirements;
import com.google.devtools.build.lib.actions.PathStripper;
import com.google.devtools.build.lib.actions.PathStripper.PathMapper;
import com.google.devtools.build.lib.actions.ResourceSetOrBuilder;
import com.google.devtools.build.lib.actions.RunfilesSupplier;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.UserExecException;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
import com.google.devtools.build.lib.analysis.config.CoreOptions;
import com.google.devtools.build.lib.analysis.config.CoreOptions.OutputPathsMode;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.server.FailureDetails;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.server.FailureDetails.StarlarkAction.Code;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import javax.annotation.Nullable;
/** A Starlark specific SpawnAction. */
public final class StarlarkAction extends SpawnAction implements ActionCacheAwareAction {
// All the inputs of the Starlark action including those listed in the unused inputs and excluding
// the shadowed action inputs
private final NestedSet<Artifact> allStarlarkActionInputs;
private final Optional<Artifact> unusedInputsList;
private final Optional<Action> shadowedAction;
private boolean inputsDiscovered = false;
/**
* Constructs a StarlarkAction using direct initialization arguments.
*
* <p>All collections provided must not be subsequently modified.
*
* @param owner the owner of the Action
* @param tools the set of files comprising the tool that does the work (e.g. compiler). This is a
* subset of "inputs" and is only used by the WorkerSpawnStrategy
* @param inputs the set of all files potentially read by this action; must not be subsequently
* modified
* @param outputs the set of all files written by this action; must not be subsequently modified.
* @param resourceSetOrBuilder the resources consumed by executing this Action
* @param commandLines the command lines to execute. This includes the main argv vector and any
* param file-backed command lines.
* @param commandLineLimits the command line limits, from the build configuration
* @param env the action's environment
* @param executionInfo out-of-band information for scheduling the spawn
* @param progressMessage the message printed during the progression of the build
* @param runfilesSupplier {@link RunfilesSupplier}s describing the runfiles for the action
* @param mnemonic the mnemonic that is reported in the master log
* @param unusedInputsList file containing the list of inputs that were not used by the action.
* @param shadowedAction the action to use its inputs and environment during execution
* @param stripOutputPaths should this action strip config prefixes from output paths? See {@link
* PathStripper} for details.
*/
private StarlarkAction(
ActionOwner owner,
NestedSet<Artifact> tools,
NestedSet<Artifact> inputs,
Iterable<Artifact> outputs,
ResourceSetOrBuilder resourceSetOrBuilder,
CommandLines commandLines,
CommandLineLimits commandLineLimits,
ActionEnvironment env,
ImmutableMap<String, String> executionInfo,
CharSequence progressMessage,
RunfilesSupplier runfilesSupplier,
String mnemonic,
Optional<Artifact> unusedInputsList,
Optional<Action> shadowedAction,
boolean stripOutputPaths) {
super(
owner,
tools,
shadowedAction.isPresent()
? createInputs(shadowedAction.get().getInputs(), inputs)
: inputs,
outputs,
resourceSetOrBuilder,
commandLines,
commandLineLimits,
env,
executionInfo,
progressMessage,
runfilesSupplier,
mnemonic,
/* resultConsumer= */ null,
stripOutputPaths);
this.allStarlarkActionInputs = inputs;
this.unusedInputsList = unusedInputsList;
this.shadowedAction = shadowedAction;
}
@VisibleForTesting
public Optional<Artifact> getUnusedInputsList() {
return unusedInputsList;
}
@Override
public boolean isShareable() {
return unusedInputsList.isEmpty();
}
@Override
public boolean discoversInputs() {
return unusedInputsList.isPresent()
|| (shadowedAction.isPresent() && shadowedAction.get().discoversInputs());
}
@Override
protected boolean inputsDiscovered() {
return inputsDiscovered;
}
@Override
protected void setInputsDiscovered(boolean inputsDiscovered) {
this.inputsDiscovered = inputsDiscovered;
}
@Override
public NestedSet<Artifact> getAllowedDerivedInputs() {
if (shadowedAction.isPresent()) {
return createInputs(shadowedAction.get().getAllowedDerivedInputs(), getInputs());
}
return getInputs();
}
/**
* This method returns null when a required SkyValue is missing and a Skyframe restart is
* required.
*/
@Nullable
@Override
public NestedSet<Artifact> discoverInputs(ActionExecutionContext actionExecutionContext)
throws ActionExecutionException, InterruptedException {
// If the Starlark action shadows another action and the shadowed action discovers its inputs,
// we get the shadowed action's discovered inputs and append it to the Starlark action inputs.
if (shadowedAction.isPresent() && shadowedAction.get().discoversInputs()) {
Action shadowedActionObj = shadowedAction.get();
NestedSet<Artifact> oldInputs = getInputs();
NestedSet<Artifact> inputFilesForExtraAction =
shadowedActionObj.getInputFilesForExtraAction(actionExecutionContext);
if (inputFilesForExtraAction == null) {
return null;
}
updateInputs(
createInputs(
shadowedActionObj.getInputs(), inputFilesForExtraAction, allStarlarkActionInputs));
return NestedSetBuilder.wrap(
Order.STABLE_ORDER, Sets.difference(getInputs().toSet(), oldInputs.toSet()));
}
// Otherwise, we need to "re-discover" all the original inputs: the unused ones that were
// removed might now be needed.
updateInputs(allStarlarkActionInputs);
return allStarlarkActionInputs;
}
private InputStream getUnusedInputListInputStream(
ActionExecutionContext actionExecutionContext, List<SpawnResult> spawnResults)
throws IOException, ExecException {
// Check if the file is in-memory.
// Note: SpawnActionContext guarantees that the first list entry exists and corresponds to the
// executed spawn.
Artifact unusedInputsListArtifact = unusedInputsList.get();
InputStream inputStream = spawnResults.get(0).getInMemoryOutput(unusedInputsListArtifact);
if (inputStream != null) {
return inputStream;
}
// Fallback to reading from disk.
try {
return actionExecutionContext
.getPathResolver()
.toPath(unusedInputsListArtifact)
.getInputStream();
} catch (FileNotFoundException e) {
String message =
"Action did not create expected output file listing unused inputs: "
+ unusedInputsListArtifact.getExecPathString();
throw new UserExecException(
e, createFailureDetail(message, Code.UNUSED_INPUT_LIST_FILE_NOT_FOUND));
}
}
@Override
protected void afterExecute(
ActionExecutionContext actionExecutionContext, List<SpawnResult> spawnResults)
throws ExecException {
if (unusedInputsList.isEmpty()) {
return;
}
// Get all the action's inputs after execution which will include the shadowed action
// discovered inputs
NestedSet<Artifact> allInputs = getInputs();
Map<String, Artifact> usedInputs = new HashMap<>();
for (Artifact input : allInputs.toList()) {
usedInputs.put(input.getExecPathString(), input);
}
try (BufferedReader br =
new BufferedReader(
new InputStreamReader(
getUnusedInputListInputStream(actionExecutionContext, spawnResults), UTF_8))) {
String line;
while ((line = br.readLine()) != null) {
line = line.trim();
if (line.isEmpty()) {
continue;
}
usedInputs.remove(line);
}
} catch (IOException e) {
throw new EnvironmentalExecException(
e,
createFailureDetail("Unused inputs read failure", Code.UNUSED_INPUT_LIST_READ_FAILURE));
}
updateInputs(NestedSetBuilder.wrap(Order.STABLE_ORDER, usedInputs.values()));
}
@Override
Spawn getSpawnForExtraAction() throws CommandLineExpansionException, InterruptedException {
if (shadowedAction.isPresent()) {
return getSpawn(createInputs(shadowedAction.get().getInputs(), allStarlarkActionInputs));
}
return getSpawn(allStarlarkActionInputs);
}
/**
* This method returns null when a required SkyValue is missing and a Skyframe restart is
* required.
*/
@Nullable
@Override
public NestedSet<Artifact> getInputFilesForExtraAction(
ActionExecutionContext actionExecutionContext)
throws ActionExecutionException, InterruptedException {
if (shadowedAction.isPresent()) {
NestedSet<Artifact> inputFilesForExtraAction =
shadowedAction.get().getInputFilesForExtraAction(actionExecutionContext);
if (inputFilesForExtraAction == null) {
return null;
}
return createInputs(
shadowedAction.get().getInputFilesForExtraAction(actionExecutionContext),
allStarlarkActionInputs);
}
return allStarlarkActionInputs;
}
private static FailureDetail createFailureDetail(String message, Code detailedCode) {
return FailureDetail.newBuilder()
.setMessage(message)
.setStarlarkAction(FailureDetails.StarlarkAction.newBuilder().setCode(detailedCode))
.build();
}
@SafeVarargs
private static NestedSet<Artifact> createInputs(NestedSet<Artifact>... inputsLists) {
NestedSetBuilder<Artifact> nestedSetBuilder = new NestedSetBuilder<>(Order.STABLE_ORDER);
for (NestedSet<Artifact> inputs : inputsLists) {
nestedSetBuilder.addTransitive(inputs);
}
return nestedSetBuilder.build();
}
/**
* StarlarkAction can contain `unused_input_list`, which rely on the action cache entry's file
* list to determine the list of inputs for a subsequent run, taking into account
* unused_input_list. Hence we need to store the inputs' execPaths in the action cache. The
* StarlarkAction inputs' execPaths should also be stored in the action cache if it shadows
* another action that discovers its inputs to avoid re-running input discovery after a shutdown.
*/
@Override
public boolean storeInputsExecPathsInActionCache() {
return unusedInputsList.isPresent()
|| (shadowedAction.isPresent() && shadowedAction.get().discoversInputs());
}
/**
* Return a spawn that is representative of the command that this Action will execute in the given
* client environment.
*
* <p>Overriding this method to add the environment of the shadowed action, if any, to the
* execution spawn.
*/
@Override
public Spawn getSpawn(ActionExecutionContext actionExecutionContext)
throws CommandLineExpansionException, InterruptedException {
return getSpawn(
actionExecutionContext.getArtifactExpander(),
getEffectiveEnvironment(actionExecutionContext.getClientEnv()),
/*envResolved=*/ true,
actionExecutionContext.getTopLevelFilesets(),
/*reportOutputs=*/ true);
}
@Override
public ImmutableMap<String, String> getEffectiveEnvironment(Map<String, String> clientEnv)
throws CommandLineExpansionException {
ActionEnvironment env = getEnvironment();
Map<String, String> environment = Maps.newLinkedHashMapWithExpectedSize(env.size());
if (shadowedAction.isPresent()) {
// Put all the variables of the shadowed action's environment
environment.putAll(shadowedAction.get().getEffectiveEnvironment(clientEnv));
}
// This order guarantees that the Starlark action can overwrite any variable in its shadowed
// action environment with a new value.
env.resolve(environment, clientEnv);
return ImmutableMap.copyOf(environment);
}
/** Builder class to construct {@link StarlarkAction} instances. */
public static class Builder extends SpawnAction.Builder {
private Optional<Artifact> unusedInputsList = Optional.empty();
private Optional<Action> shadowedAction = Optional.empty();
@CanIgnoreReturnValue
public Builder setUnusedInputsList(Optional<Artifact> unusedInputsList) {
this.unusedInputsList = unusedInputsList;
return this;
}
@CanIgnoreReturnValue
public Builder setShadowedAction(Optional<Action> shadowedAction) {
this.shadowedAction = shadowedAction;
return this;
}
/** Creates a SpawnAction. */
@Override
protected SpawnAction createSpawnAction(
ActionOwner owner,
NestedSet<Artifact> tools,
NestedSet<Artifact> inputsAndTools,
ImmutableSet<Artifact> outputs,
ResourceSetOrBuilder resourceSetOrBuilder,
CommandLines commandLines,
CommandLineLimits commandLineLimits,
ActionEnvironment env,
@Nullable BuildConfigurationValue configuration,
ImmutableMap<String, String> executionInfo,
CharSequence progressMessage,
RunfilesSupplier runfilesSupplier,
String mnemonic) {
if (unusedInputsList.isPresent()) {
// Always download unused_inputs_list file from remote cache.
executionInfo =
ImmutableMap.<String, String>builderWithExpectedSize(executionInfo.size() + 1)
.putAll(executionInfo)
.put(
ExecutionRequirements.REMOTE_EXECUTION_INLINE_OUTPUTS,
unusedInputsList.get().getExecPathString())
.buildOrThrow();
}
return new StarlarkAction(
owner,
tools,
inputsAndTools,
outputs,
resourceSetOrBuilder,
commandLines,
commandLineLimits,
env,
executionInfo,
progressMessage,
runfilesSupplier,
mnemonic,
unusedInputsList,
shadowedAction,
stripOutputPaths(mnemonic, inputsAndTools, outputs, configuration));
}
/**
* Should we strip output paths for this action?
*
* <p>Since we don't currently have a proper Starlark API for this, we hard-code support for
* specific actions we're interested in.
*
* <p>The difference between this and {@link PathMapper#mapCustomStarlarkArgs} is this triggers
* Bazel's standard path stripping logic for chosen mnemonics while {@link
* PathMapper#mapCustomStarlarkArgs} custom-adjusts certain command line parameters the standard
* logic can't handle. For example, Starlark rules that only set {@code
* ctx.actions.args().add(file_handle)} need an entry here but not in {@link
* PathMapper#mapCustomStarlarkArgs} because standard path stripping logic handles that
* interface.
*/
private static boolean stripOutputPaths(
String mnemonic,
NestedSet<Artifact> inputs,
ImmutableSet<Artifact> outputs,
BuildConfigurationValue configuration) {
ImmutableList<String> qualifyingMnemonics =
ImmutableList.of(
"AndroidLint",
"ParseAndroidResources",
"MergeAndroidAssets",
"LinkAndroidResources",
"CompileAndroidResources",
"StarlarkMergeCompiledAndroidResources",
"MergeManifests",
"StarlarkRClassGenerator",
"StarlarkAARGenerator",
"Desugar",
"Jetify",
"DeJetify",
"JetifySrcs",
"DejetifySrcs");
CoreOptions coreOptions = configuration.getOptions().get(CoreOptions.class);
return coreOptions.outputPathsMode == OutputPathsMode.STRIP
&& qualifyingMnemonics.contains(mnemonic)
&& PathStripper.isPathStrippable(
inputs, Iterables.get(outputs, 0).getExecPath().subFragment(0, 1));
}
}
}