Skip to content

Commit

Permalink
Avoid adding layers for buildpacks that exist in the builder
Browse files Browse the repository at this point in the history
This commit adds validation of any buildpacks that are specified for
image building to match them against buildpacks that are bundled in
the builder. If an image buildpack's ID, version, and one layer
hash match the same information stored in a label on the builder
image, that buildpack won't be added and the buildpack bundled in
the builder will be used instead. This reduces the chance of adding to
the total count of layers in a builder image unnecessarily.

Fixes gh-31233
  • Loading branch information
scottfrederick committed Jun 30, 2022
1 parent 6411f88 commit 17bdc52
Show file tree
Hide file tree
Showing 8 changed files with 442 additions and 18 deletions.
@@ -1,5 +1,5 @@
/*
* Copyright 2012-2021 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -105,7 +105,8 @@ public void build(BuildRequest request) throws DockerEngineException, IOExceptio
Image runImage = imageFetcher.fetchImage(ImageType.RUNNER, request.getRunImage());
assertStackIdsMatch(runImage, builderImage);
BuildOwner buildOwner = BuildOwner.fromEnv(builderImage.getConfig().getEnv());
Buildpacks buildpacks = getBuildpacks(request, imageFetcher, builderMetadata);
BuildpackLayersMetadata buildpackLayersMetadata = BuildpackLayersMetadata.fromImage(builderImage);
Buildpacks buildpacks = getBuildpacks(request, imageFetcher, builderMetadata, buildpackLayersMetadata);
EphemeralBuilder ephemeralBuilder = new EphemeralBuilder(buildOwner, builderImage, request.getName(),
builderMetadata, request.getCreator(), request.getEnv(), buildpacks);
this.docker.image().load(ephemeralBuilder.getArchive(), UpdateListener.none());
Expand Down Expand Up @@ -141,8 +142,10 @@ private void assertStackIdsMatch(Image runImage, Image builderImage) {
+ "' does not match builder stack '" + builderImageStackId + "'");
}

private Buildpacks getBuildpacks(BuildRequest request, ImageFetcher imageFetcher, BuilderMetadata builderMetadata) {
BuildpackResolverContext resolverContext = new BuilderResolverContext(imageFetcher, builderMetadata);
private Buildpacks getBuildpacks(BuildRequest request, ImageFetcher imageFetcher, BuilderMetadata builderMetadata,
BuildpackLayersMetadata buildpackLayersMetadata) {
BuildpackResolverContext resolverContext = new BuilderResolverContext(imageFetcher, builderMetadata,
buildpackLayersMetadata);
return BuildpackResolvers.resolveAll(resolverContext, request.getBuildpacks());
}

Expand Down Expand Up @@ -239,16 +242,25 @@ private class BuilderResolverContext implements BuildpackResolverContext {

private final BuilderMetadata builderMetadata;

BuilderResolverContext(ImageFetcher imageFetcher, BuilderMetadata builderMetadata) {
private final BuildpackLayersMetadata buildpackLayersMetadata;

BuilderResolverContext(ImageFetcher imageFetcher, BuilderMetadata builderMetadata,
BuildpackLayersMetadata buildpackLayersMetadata) {
this.imageFetcher = imageFetcher;
this.builderMetadata = builderMetadata;
this.buildpackLayersMetadata = buildpackLayersMetadata;
}

@Override
public List<BuildpackMetadata> getBuildpackMetadata() {
return this.builderMetadata.getBuildpacks();
}

@Override
public BuildpackLayersMetadata getBuildpackLayersMetadata() {
return this.buildpackLayersMetadata;
}

@Override
public Image fetchImage(ImageReference reference, ImageType imageType) throws IOException {
return this.imageFetcher.fetchImage(imageType, reference);
Expand Down
@@ -0,0 +1,194 @@
/*
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.springframework.boot.buildpack.platform.build;

import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.HashMap;
import java.util.Map;

import com.fasterxml.jackson.databind.JsonNode;

import org.springframework.boot.buildpack.platform.docker.type.Image;
import org.springframework.boot.buildpack.platform.docker.type.ImageConfig;
import org.springframework.boot.buildpack.platform.json.MappedObject;
import org.springframework.boot.buildpack.platform.json.SharedObjectMapper;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;

/**
* Buildpack layers metadata information.
*
* @author Scott Frederick
*/
final class BuildpackLayersMetadata extends MappedObject {

private static final String LABEL_NAME = "io.buildpacks.buildpack.layers";

private final Buildpacks buildpacks;

private BuildpackLayersMetadata(JsonNode node) {
super(node, MethodHandles.lookup());
this.buildpacks = Buildpacks.fromJson(getNode());
}

/**
* Return the metadata details of a buildpack with the given ID and version.
* @param id the buildpack ID
* @param version the buildpack version
* @return the buildpack details or {@code null} if a buildpack with the given ID and
* version does not exist in the metadata
*/
BuildpackLayerDetails getBuildpack(String id, String version) {
return this.buildpacks.getBuildpack(id, version);
}

/**
* Create a {@link BuildpackLayersMetadata} from an image.
* @param image the source image
* @return the buildpack layers metadata
* @throws IOException on IO error
*/
static BuildpackLayersMetadata fromImage(Image image) throws IOException {
Assert.notNull(image, "Image must not be null");
return fromImageConfig(image.getConfig());
}

/**
* Create a {@link BuildpackLayersMetadata} from image config.
* @param imageConfig the source image config
* @return the buildpack layers metadata
* @throws IOException on IO error
*/
static BuildpackLayersMetadata fromImageConfig(ImageConfig imageConfig) throws IOException {
Assert.notNull(imageConfig, "ImageConfig must not be null");
String json = imageConfig.getLabels().get(LABEL_NAME);
Assert.notNull(json, () -> "No '" + LABEL_NAME + "' label found in image config labels '"
+ StringUtils.collectionToCommaDelimitedString(imageConfig.getLabels().keySet()) + "'");
return fromJson(json);
}

/**
* Create a {@link BuildpackLayersMetadata} from JSON.
* @param json the source JSON
* @return the buildpack layers metadata
* @throws IOException on IO error
*/
static BuildpackLayersMetadata fromJson(String json) throws IOException {
return fromJson(SharedObjectMapper.get().readTree(json));
}

/**
* Create a {@link BuildpackLayersMetadata} from JSON.
* @param node the source JSON
* @return the buildpack layers metadata
*/
static BuildpackLayersMetadata fromJson(JsonNode node) {
return new BuildpackLayersMetadata(node);
}

private static class Buildpacks {

private final Map<String, BuildpackVersions> buildpacks = new HashMap<>();

private BuildpackLayerDetails getBuildpack(String id, String version) {
if (this.buildpacks.containsKey(id)) {
return this.buildpacks.get(id).getBuildpack(version);
}
return null;
}

private void addBuildpackVersions(String id, BuildpackVersions versions) {
this.buildpacks.put(id, versions);
}

private static Buildpacks fromJson(JsonNode node) {
Buildpacks buildpacks = new Buildpacks();
node.fields().forEachRemaining((field) -> buildpacks.addBuildpackVersions(field.getKey(),
BuildpackVersions.fromJson(field.getValue())));
return buildpacks;
}

}

private static class BuildpackVersions {

private final Map<String, BuildpackLayerDetails> versions = new HashMap<>();

private BuildpackLayerDetails getBuildpack(String version) {
return this.versions.get(version);
}

private void addBuildpackVersion(String version, BuildpackLayerDetails details) {
this.versions.put(version, details);
}

private static BuildpackVersions fromJson(JsonNode node) {
BuildpackVersions versions = new BuildpackVersions();
node.fields().forEachRemaining((field) -> versions.addBuildpackVersion(field.getKey(),
BuildpackLayerDetails.fromJson(field.getValue())));
return versions;
}

}

static final class BuildpackLayerDetails extends MappedObject {

private final String name;

private final String homepage;

private final String layerDiffId;

private BuildpackLayerDetails(JsonNode node) {
super(node, MethodHandles.lookup());
this.name = valueAt("/name", String.class);
this.homepage = valueAt("/homepage", String.class);
this.layerDiffId = valueAt("/layerDiffID", String.class);
}

/**
* Return the buildpack name.
* @return the name
*/
String getName() {
return this.name;
}

/**
* Return the buildpack homepage address.
* @return the homepage address
*/
String getHomepage() {
return this.homepage;
}

/**
* Return the buildpack layer {@code diffID}.
* @return the layer {@code diffID}
*/
String getLayerDiffId() {
return this.layerDiffId;
}

private static BuildpackLayerDetails fromJson(JsonNode node) {
return new BuildpackLayerDetails(node);
}

}

}
@@ -1,5 +1,5 @@
/*
* Copyright 2012-2021 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -34,6 +34,8 @@ interface BuildpackResolverContext {

List<BuildpackMetadata> getBuildpackMetadata();

BuildpackLayersMetadata getBuildpackLayersMetadata();

/**
* Retrieve an image.
* @param reference the image reference
Expand Down
@@ -1,5 +1,5 @@
/*
* Copyright 2012-2021 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -28,10 +28,12 @@
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;

import org.springframework.boot.buildpack.platform.build.BuildpackLayersMetadata.BuildpackLayerDetails;
import org.springframework.boot.buildpack.platform.docker.transport.DockerEngineException;
import org.springframework.boot.buildpack.platform.docker.type.Image;
import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
import org.springframework.boot.buildpack.platform.docker.type.Layer;
import org.springframework.boot.buildpack.platform.docker.type.LayerId;
import org.springframework.boot.buildpack.platform.io.IOConsumer;
import org.springframework.boot.buildpack.platform.io.TarArchive;
import org.springframework.util.StreamUtils;
Expand Down Expand Up @@ -59,21 +61,38 @@ private ImageBuildpack(BuildpackResolverContext context, ImageReference imageRef
Image image = context.fetchImage(reference, ImageType.BUILDPACK);
BuildpackMetadata buildpackMetadata = BuildpackMetadata.fromImage(image);
this.coordinates = BuildpackCoordinates.fromBuildpackMetadata(buildpackMetadata);
this.exportedLayers = new ExportedLayers(context, reference);
if (!buildpackExistsInBuilder(context, image.getLayers())) {
this.exportedLayers = new ExportedLayers(context, reference);
}
else {
this.exportedLayers = null;
}
}
catch (IOException | DockerEngineException ex) {
throw new IllegalArgumentException("Error pulling buildpack image '" + reference + "'", ex);
}
}

private boolean buildpackExistsInBuilder(BuildpackResolverContext context, List<LayerId> imageLayers) {
BuildpackLayerDetails buildpackLayerDetails = context.getBuildpackLayersMetadata()
.getBuildpack(this.coordinates.getId(), this.coordinates.getVersion());
if (buildpackLayerDetails != null) {
String layerDiffId = buildpackLayerDetails.getLayerDiffId();
return imageLayers.stream().map(LayerId::toString).anyMatch((layerId) -> layerId.equals(layerDiffId));
}
return false;
}

@Override
public BuildpackCoordinates getCoordinates() {
return this.coordinates;
}

@Override
public void apply(IOConsumer<Layer> layers) throws IOException {
this.exportedLayers.apply(layers);
if (this.exportedLayers != null) {
this.exportedLayers.apply(layers);
}
}

/**
Expand Down

0 comments on commit 17bdc52

Please sign in to comment.