Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
repository: Add a Resource cache to FileSetRepository
The cache reduces the need to create new Resource objects, including SHA-256 computation, for unchanged files. Fixes #5367 Signed-off-by: BJ Hargrave <bj@hargrave.dev>
- Loading branch information
1 parent
e1eefb7
commit e6b04ec
Showing
4 changed files
with
293 additions
and
54 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
93 changes: 93 additions & 0 deletions
93
biz.aQute.repository/src/aQute/bnd/repository/fileset/ResourceCache.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| package aQute.bnd.repository.fileset; | ||
|
|
||
| import static aQute.bnd.exceptions.FunctionWithException.asFunctionOrElse; | ||
|
|
||
| import java.io.File; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
| import java.util.concurrent.ConcurrentHashMap; | ||
| import java.util.concurrent.TimeUnit; | ||
|
|
||
| import org.osgi.framework.namespace.IdentityNamespace; | ||
| import org.osgi.resource.Resource; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import aQute.bnd.osgi.Jar; | ||
| import aQute.bnd.osgi.resource.CapReqBuilder; | ||
| import aQute.bnd.osgi.resource.ResourceBuilder; | ||
| import aQute.bnd.service.RepositoryPlugin; | ||
| import aQute.bnd.version.MavenVersion; | ||
| import aQute.bnd.version.Version; | ||
| import aQute.maven.api.Revision; | ||
| import aQute.maven.provider.POM; | ||
|
|
||
| final class ResourceCache { | ||
| private final static Logger logger = LoggerFactory | ||
| .getLogger(ResourceCache.class); | ||
| private final static long EXPIRED_DURATION_NANOS = TimeUnit.NANOSECONDS.convert(30L, | ||
| TimeUnit.MINUTES); | ||
| private final Map<ResourceCacheKey, Resource> cache; | ||
| private long time; | ||
|
|
||
| ResourceCache() { | ||
| cache = new ConcurrentHashMap<>(); | ||
| time = System.nanoTime(); | ||
| } | ||
|
|
||
| Resource getResource(RepositoryPlugin repo, File file) { | ||
| if (!file.isFile()) { | ||
| return null; | ||
| } | ||
| // Make sure we don't grow infinitely | ||
| final long now = System.nanoTime(); | ||
| if ((now - time) > EXPIRED_DURATION_NANOS) { | ||
| cache.keySet() | ||
| .removeIf(key -> (now - key.time) > EXPIRED_DURATION_NANOS); | ||
| time = now; | ||
| } | ||
| ResourceCacheKey cacheKey = new ResourceCacheKey(file); | ||
| Resource resource = cache.computeIfAbsent(cacheKey, key -> { | ||
| ResourceBuilder rb = new ResourceBuilder(); | ||
| try { | ||
| boolean hasIdentity = rb.addFile(file, null); | ||
| if (!hasIdentity) { | ||
| try (Jar jar = new Jar(file)) { | ||
| Optional<Revision> revision = jar.getPomXmlResources() | ||
| .findFirst() | ||
| .map(asFunctionOrElse(pomResource -> new POM(null, pomResource.openInputStream(), true), | ||
| null)) | ||
| .map(POM::getRevision); | ||
|
|
||
| String name = jar.getModuleName(); | ||
| if (name == null) { | ||
| name = revision.map(r -> r.program.toString()) | ||
| .orElse(null); | ||
| if (name == null) { | ||
| return null; | ||
| } | ||
| } | ||
|
|
||
| Version version = revision.map(r -> r.version.getOSGiVersion()) | ||
| .orElse(null); | ||
| if (version == null) { | ||
| version = new MavenVersion(jar.getModuleVersion()).getOSGiVersion(); | ||
| } | ||
|
|
||
| CapReqBuilder identity = new CapReqBuilder(IdentityNamespace.IDENTITY_NAMESPACE) | ||
| .addAttribute(IdentityNamespace.IDENTITY_NAMESPACE, name) | ||
| .addAttribute(IdentityNamespace.CAPABILITY_VERSION_ATTRIBUTE, version) | ||
| .addAttribute(IdentityNamespace.CAPABILITY_TYPE_ATTRIBUTE, IdentityNamespace.TYPE_UNKNOWN); | ||
| rb.addCapability(identity); | ||
| } | ||
| } | ||
| } catch (Exception f) { | ||
| return null; | ||
| } | ||
| logger.debug("{}: parsing {}", repo.getName(), file); | ||
| return rb.build(); | ||
| }); | ||
|
|
||
| return resource; | ||
| } | ||
| } |
68 changes: 68 additions & 0 deletions
68
biz.aQute.repository/src/aQute/bnd/repository/fileset/ResourceCacheKey.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| package aQute.bnd.repository.fileset; | ||
|
|
||
| import java.io.File; | ||
| import java.io.IOException; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
| import java.nio.file.attribute.BasicFileAttributeView; | ||
| import java.nio.file.attribute.BasicFileAttributes; | ||
| import java.nio.file.attribute.FileTime; | ||
| import java.util.Objects; | ||
| import java.util.Optional; | ||
|
|
||
| import aQute.bnd.exceptions.Exceptions; | ||
|
|
||
| final class ResourceCacheKey { | ||
| private final Path path; | ||
| private final Object fileKey; | ||
| private final FileTime lastModifiedTime; | ||
| private final long size; | ||
| final long time; | ||
|
|
||
| ResourceCacheKey(File file) { | ||
| this(file.toPath()); | ||
| } | ||
|
|
||
| ResourceCacheKey(Path path) { | ||
| path = path.toAbsolutePath(); | ||
| BasicFileAttributes attributes; | ||
| try { | ||
| attributes = Files.getFileAttributeView(path, BasicFileAttributeView.class) | ||
| .readAttributes(); | ||
| } catch (IOException e) { | ||
| throw Exceptions.duck(e); | ||
| } | ||
| if (!attributes.isRegularFile()) { | ||
| throw new IllegalArgumentException("File must be a regular file: " + path); | ||
| } | ||
| this.path = path; | ||
| this.fileKey = Optional.ofNullable(attributes.fileKey()) | ||
| .orElse(path); // Windows | ||
| this.lastModifiedTime = attributes.lastModifiedTime(); | ||
| this.size = attributes.size(); | ||
| this.time = System.nanoTime(); | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| return (Objects.hashCode(fileKey) * 31 + Objects.hashCode(lastModifiedTime)) * 31 + Long.hashCode(size); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object obj) { | ||
| if (this == obj) { | ||
| return true; | ||
| } | ||
| if (!(obj instanceof ResourceCacheKey)) { | ||
| return false; | ||
| } | ||
| ResourceCacheKey other = (ResourceCacheKey) obj; | ||
| return Objects.equals(fileKey, other.fileKey) && Objects.equals(lastModifiedTime, other.lastModifiedTime) | ||
| && (size == other.size); | ||
| } | ||
|
|
||
| @Override | ||
| public String toString() { | ||
| return path.toString(); | ||
| } | ||
| } |
126 changes: 126 additions & 0 deletions
126
biz.aQute.repository/test/aQute/bnd/repository/fileset/RepositoryCacheKeyTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,126 @@ | ||
| package aQute.bnd.repository.fileset; | ||
|
|
||
| import static org.assertj.core.api.Assertions.assertThatObject; | ||
|
|
||
| import java.nio.charset.StandardCharsets; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
| import java.nio.file.attribute.BasicFileAttributeView; | ||
| import java.nio.file.attribute.BasicFileAttributes; | ||
| import java.nio.file.attribute.FileTime; | ||
| import java.time.Instant; | ||
|
|
||
| import org.junit.jupiter.api.Test; | ||
| import org.junit.jupiter.api.condition.DisabledOnOs; | ||
| import org.junit.jupiter.api.condition.OS; | ||
|
|
||
| import aQute.bnd.test.jupiter.InjectTemporaryDirectory; | ||
| import aQute.lib.io.IO; | ||
|
|
||
| class RepositoryCacheKeyTest { | ||
|
|
||
| @Test | ||
| void unchanged(@InjectTemporaryDirectory | ||
| Path tmp) throws Exception { | ||
| Path subject = tmp.resolve("test"); | ||
| IO.store("line1", subject, StandardCharsets.UTF_8); | ||
| ResourceCacheKey key1 = new ResourceCacheKey(subject); | ||
| ResourceCacheKey key2 = new ResourceCacheKey(subject); | ||
| assertThatObject(key1).isEqualTo(key2); | ||
| assertThatObject(key1).hasSameHashCodeAs(key2); | ||
| } | ||
|
|
||
| @Test | ||
| void change_modified(@InjectTemporaryDirectory | ||
| Path tmp) throws Exception { | ||
| Path subject = tmp.resolve("test"); | ||
| IO.store("line1", subject, StandardCharsets.UTF_8); | ||
| ResourceCacheKey key1 = new ResourceCacheKey(subject); | ||
| BasicFileAttributes attributes = Files.getFileAttributeView(subject, BasicFileAttributeView.class) | ||
| .readAttributes(); | ||
| FileTime lastModifiedTime = attributes.lastModifiedTime(); | ||
| Instant plusSeconds = lastModifiedTime.toInstant() | ||
| .plusSeconds(10L); | ||
| Files.setLastModifiedTime(subject, FileTime.from(plusSeconds)); | ||
| ResourceCacheKey key2 = new ResourceCacheKey(subject); | ||
| assertThatObject(key1).isNotEqualTo(key2); | ||
| assertThatObject(key1).doesNotHaveSameHashCodeAs(key2); | ||
| } | ||
|
|
||
| @Test | ||
| void change_size(@InjectTemporaryDirectory | ||
| Path tmp) throws Exception { | ||
| Path subject = tmp.resolve("test"); | ||
| IO.store("line1", subject, StandardCharsets.UTF_8); | ||
| ResourceCacheKey key1 = new ResourceCacheKey(subject); | ||
| BasicFileAttributes attributes = Files.getFileAttributeView(subject, BasicFileAttributeView.class) | ||
| .readAttributes(); | ||
| FileTime lastModifiedTime = attributes.lastModifiedTime(); | ||
| IO.store("line100", subject, StandardCharsets.UTF_8); | ||
| Files.setLastModifiedTime(subject, lastModifiedTime); | ||
| ResourceCacheKey key2 = new ResourceCacheKey(subject); | ||
| assertThatObject(key1).isNotEqualTo(key2); | ||
| assertThatObject(key1).doesNotHaveSameHashCodeAs(key2); | ||
| } | ||
|
|
||
| @DisabledOnOs(value = OS.WINDOWS, disabledReason = "Windows FS does not support fileKey") | ||
| @Test | ||
| void change_filekey(@InjectTemporaryDirectory | ||
| Path tmp) throws Exception { | ||
| Path subject = tmp.resolve("test"); | ||
| IO.store("line1", subject, StandardCharsets.UTF_8); | ||
| ResourceCacheKey key1 = new ResourceCacheKey(subject); | ||
| BasicFileAttributes attributes = Files.getFileAttributeView(subject, BasicFileAttributeView.class) | ||
| .readAttributes(); | ||
| assertThatObject(attributes.fileKey()).isNotNull(); | ||
| FileTime lastModifiedTime = attributes.lastModifiedTime(); | ||
| Path subject2 = tmp.resolve("test.tmp"); | ||
| IO.store("line2", subject2, StandardCharsets.UTF_8); | ||
| Files.setLastModifiedTime(subject2, lastModifiedTime); | ||
| IO.rename(subject2, subject); | ||
| ResourceCacheKey key2 = new ResourceCacheKey(subject); | ||
| attributes = Files.getFileAttributeView(subject, BasicFileAttributeView.class) | ||
| .readAttributes(); | ||
| assertThatObject(attributes.fileKey()).isNotNull(); | ||
| assertThatObject(key1).as("key2 not equal") | ||
| .isNotEqualTo(key2); | ||
| assertThatObject(key1).as("key2 different hash") | ||
| .doesNotHaveSameHashCodeAs(key2); | ||
| } | ||
|
|
||
| @Test | ||
| void change_file_modified(@InjectTemporaryDirectory | ||
| Path tmp) throws Exception { | ||
| Path subject = tmp.resolve("test"); | ||
| IO.store("line1", subject, StandardCharsets.UTF_8); | ||
| ResourceCacheKey key1 = new ResourceCacheKey(subject); | ||
| Path subject2 = tmp.resolve("test.tmp"); | ||
| IO.store("line2", subject2, StandardCharsets.UTF_8); | ||
| BasicFileAttributes attributes = Files.getFileAttributeView(subject2, BasicFileAttributeView.class) | ||
| .readAttributes(); | ||
| FileTime lastModifiedTime = attributes.lastModifiedTime(); | ||
| Instant plusSeconds = lastModifiedTime.toInstant() | ||
| .plusSeconds(10L); | ||
| Files.setLastModifiedTime(subject2, FileTime.from(plusSeconds)); | ||
| IO.rename(subject2, subject); | ||
| ResourceCacheKey key2 = new ResourceCacheKey(subject); | ||
| assertThatObject(key1).as("key2 not equal") | ||
| .isNotEqualTo(key2); | ||
| assertThatObject(key1).as("key2 different hash") | ||
| .doesNotHaveSameHashCodeAs(key2); | ||
| } | ||
|
|
||
| @Test | ||
| void different_files(@InjectTemporaryDirectory | ||
| Path tmp) throws Exception { | ||
| Path subject1 = tmp.resolve("test1"); | ||
| IO.store("line1", subject1, StandardCharsets.UTF_8); | ||
| ResourceCacheKey key1 = new ResourceCacheKey(subject1); | ||
| Path subject2 = tmp.resolve("test2"); | ||
| IO.copy(subject1, subject2); | ||
| ResourceCacheKey key2 = new ResourceCacheKey(subject2); | ||
| assertThatObject(key1).isNotEqualTo(key2); | ||
| assertThatObject(key1).doesNotHaveSameHashCodeAs(key2); | ||
| } | ||
|
|
||
| } |