From 342f9b87ab6b5635d81019b79f58b10b85c3ee6e Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Thu, 22 Jun 2023 11:07:50 +0200 Subject: [PATCH] Do not ivalidate the cache entirely on yarn3 lock file change --- __tests__/cache-utils.test.ts | 5 ++- dist/cache-save/index.js | 60 ++++++++++++++++++++++++++++- dist/setup/index.js | 67 +++++++++++++++++++++++++++++++-- src/cache-restore.ts | 9 ++++- src/cache-utils.ts | 71 +++++++++++++++++++++++++++++++++++ 5 files changed, 205 insertions(+), 7 deletions(-) diff --git a/__tests__/cache-utils.test.ts b/__tests__/cache-utils.test.ts index 56f46425a..39f376f21 100644 --- a/__tests__/cache-utils.test.ts +++ b/__tests__/cache-utils.test.ts @@ -6,7 +6,8 @@ import { PackageManagerInfo, isCacheFeatureAvailable, supportedPackageManagers, - getCommandOutput + getCommandOutput, + resetProjectDirectoriesMemoized } from '../src/cache-utils'; import fs from 'fs'; import * as cacheUtils from '../src/cache-utils'; @@ -103,6 +104,8 @@ describe('cache-utils', () => { (pattern: string): Promise => MockGlobber.create(['/foo', '/bar']) ); + + resetProjectDirectoriesMemoized(); }); afterEach(() => { diff --git a/dist/cache-save/index.js b/dist/cache-save/index.js index 3a8d0cee2..4f0a1b625 100644 --- a/dist/cache-save/index.js +++ b/dist/cache-save/index.js @@ -60434,7 +60434,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.isCacheFeatureAvailable = exports.isGhes = exports.getCacheDirectories = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0; +exports.isCacheFeatureAvailable = exports.isGhes = exports.repoHasYarn3ManagedCache = exports.getCacheDirectories = exports.resetProjectDirectoriesMemoized = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0; const core = __importStar(__nccwpck_require__(2186)); const exec = __importStar(__nccwpck_require__(1514)); const cache = __importStar(__nccwpck_require__(7799)); @@ -60503,6 +60503,19 @@ const getPackageManagerInfo = (packageManager) => __awaiter(void 0, void 0, void } }); exports.getPackageManagerInfo = getPackageManagerInfo; +/** + * getProjectDirectoriesFromCacheDependencyPath is called twice during `restoreCache` + * - first through `getCacheDirectories` + * - second from `repoHasYarn3ManagedCache` + * + * it contains expensive IO operation and thus should be memoized + */ +let projectDirectoriesMemoized = null; +/** + * unit test must reset memoized variables + */ +const resetProjectDirectoriesMemoized = () => (projectDirectoriesMemoized = null); +exports.resetProjectDirectoriesMemoized = resetProjectDirectoriesMemoized; /** * Expands (converts) the string input `cache-dependency-path` to list of directories that * may be project roots @@ -60511,6 +60524,9 @@ exports.getPackageManagerInfo = getPackageManagerInfo; * @return list of directories and possible */ const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () { + if (projectDirectoriesMemoized !== null) { + return projectDirectoriesMemoized; + } const globber = yield glob.create(cacheDependencyPath); const cacheDependenciesPaths = yield globber.glob(); const existingDirectories = cacheDependenciesPaths @@ -60519,6 +60535,7 @@ const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __ .filter(directory => fs_1.default.lstatSync(directory).isDirectory()); if (!existingDirectories.length) core.warning(`No existing directories found containing cache-dependency-path="${cacheDependencyPath}"`); + projectDirectoriesMemoized = existingDirectories; return existingDirectories; }); /** @@ -60565,6 +60582,47 @@ const getCacheDirectories = (packageManagerInfo, cacheDependencyPath) => __await return getCacheDirectoriesForRootProject(packageManagerInfo); }); exports.getCacheDirectories = getCacheDirectories; +/** + * A function to check if the directory is a yarn project configured to manage + * obsolete dependencies in the local cache + * @param directory - a path to the folder + * @return - true if the directory's project is yarn managed + * - if there's .yarn/cache folder do not mess with the dependencies kept in the repo, return false + * - global cache is not managed by yarn @see https://yarnpkg.com/features/offline-cache, return false + * - if local cache is not explicitly enabled (not yarn3), return false + * - return true otherwise + */ +const isCacheManagedByYarn3 = (directory) => __awaiter(void 0, void 0, void 0, function* () { + const workDir = directory || process.env.GITHUB_WORKSPACE || '.'; + // if .yarn/cache directory exists the cache is managed by version control system + const yarnCacheFile = path_1.default.join(workDir, '.yarn', 'cache'); + if (fs_1.default.existsSync(yarnCacheFile) && fs_1.default.lstatSync(yarnCacheFile).isDirectory()) + return Promise.resolve(false); + // NOTE: yarn1 returns 'undefined' with rc = 0 + const enableGlobalCache = yield exports.getCommandOutput('yarn config get enableGlobalCache', workDir); + // only local cache is not managed by yarn + return enableGlobalCache === 'false'; +}); +/** + * A function to report either the repo contains at least one Yarn managed directory + * @param packageManagerInfo - used to make sure current package manager is yarn + * @return - true if there's at least one Yarn managed directory in the repo + */ +const repoHasYarn3ManagedCache = (packageManagerInfo) => __awaiter(void 0, void 0, void 0, function* () { + if (packageManagerInfo.name !== 'yarn') + return false; + const cacheDependencyPath = core.getInput('cache-dependency-path'); + const yarnDirs = cacheDependencyPath + ? yield getProjectDirectoriesFromCacheDependencyPath(cacheDependencyPath) + : ['']; + for (const dir of yarnDirs.length === 0 ? [''] : yarnDirs) { + if (yield isCacheManagedByYarn3(dir)) { + return true; + } + } + return false; +}); +exports.repoHasYarn3ManagedCache = repoHasYarn3ManagedCache; function isGhes() { const ghUrl = new URL(process.env['GITHUB_SERVER_URL'] || 'https://github.com'); return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM'; diff --git a/dist/setup/index.js b/dist/setup/index.js index 83bf5f6a3..fb63a8078 100644 --- a/dist/setup/index.js +++ b/dist/setup/index.js @@ -71153,10 +71153,13 @@ const restoreCache = (packageManager, cacheDependencyPath) => __awaiter(void 0, if (!fileHash) { throw new Error('Some specified paths were not resolved, unable to cache dependencies.'); } - const primaryKey = `node-cache-${platform}-${packageManager}-${fileHash}`; + const keyPrefix = `node-cache-${platform}-${packageManager}`; + const primaryKey = `${keyPrefix}-${fileHash}`; core.debug(`primary key is ${primaryKey}`); core.saveState(constants_1.State.CachePrimaryKey, primaryKey); - const cacheKey = yield cache.restoreCache(cachePaths, primaryKey); + const cacheKey = (yield cache_utils_1.repoHasYarn3ManagedCache(packageManagerInfo)) + ? yield cache.restoreCache(cachePaths, primaryKey, [keyPrefix]) + : yield cache.restoreCache(cachePaths, primaryKey); core.setOutput('cache-hit', Boolean(cacheKey)); if (!cacheKey) { core.info(`${packageManager} cache is not found`); @@ -71217,7 +71220,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.isCacheFeatureAvailable = exports.isGhes = exports.getCacheDirectories = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0; +exports.isCacheFeatureAvailable = exports.isGhes = exports.repoHasYarn3ManagedCache = exports.getCacheDirectories = exports.resetProjectDirectoriesMemoized = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0; const core = __importStar(__nccwpck_require__(2186)); const exec = __importStar(__nccwpck_require__(1514)); const cache = __importStar(__nccwpck_require__(7799)); @@ -71286,6 +71289,19 @@ const getPackageManagerInfo = (packageManager) => __awaiter(void 0, void 0, void } }); exports.getPackageManagerInfo = getPackageManagerInfo; +/** + * getProjectDirectoriesFromCacheDependencyPath is called twice during `restoreCache` + * - first through `getCacheDirectories` + * - second from `repoHasYarn3ManagedCache` + * + * it contains expensive IO operation and thus should be memoized + */ +let projectDirectoriesMemoized = null; +/** + * unit test must reset memoized variables + */ +const resetProjectDirectoriesMemoized = () => (projectDirectoriesMemoized = null); +exports.resetProjectDirectoriesMemoized = resetProjectDirectoriesMemoized; /** * Expands (converts) the string input `cache-dependency-path` to list of directories that * may be project roots @@ -71294,6 +71310,9 @@ exports.getPackageManagerInfo = getPackageManagerInfo; * @return list of directories and possible */ const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () { + if (projectDirectoriesMemoized !== null) { + return projectDirectoriesMemoized; + } const globber = yield glob.create(cacheDependencyPath); const cacheDependenciesPaths = yield globber.glob(); const existingDirectories = cacheDependenciesPaths @@ -71302,6 +71321,7 @@ const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __ .filter(directory => fs_1.default.lstatSync(directory).isDirectory()); if (!existingDirectories.length) core.warning(`No existing directories found containing cache-dependency-path="${cacheDependencyPath}"`); + projectDirectoriesMemoized = existingDirectories; return existingDirectories; }); /** @@ -71348,6 +71368,47 @@ const getCacheDirectories = (packageManagerInfo, cacheDependencyPath) => __await return getCacheDirectoriesForRootProject(packageManagerInfo); }); exports.getCacheDirectories = getCacheDirectories; +/** + * A function to check if the directory is a yarn project configured to manage + * obsolete dependencies in the local cache + * @param directory - a path to the folder + * @return - true if the directory's project is yarn managed + * - if there's .yarn/cache folder do not mess with the dependencies kept in the repo, return false + * - global cache is not managed by yarn @see https://yarnpkg.com/features/offline-cache, return false + * - if local cache is not explicitly enabled (not yarn3), return false + * - return true otherwise + */ +const isCacheManagedByYarn3 = (directory) => __awaiter(void 0, void 0, void 0, function* () { + const workDir = directory || process.env.GITHUB_WORKSPACE || '.'; + // if .yarn/cache directory exists the cache is managed by version control system + const yarnCacheFile = path_1.default.join(workDir, '.yarn', 'cache'); + if (fs_1.default.existsSync(yarnCacheFile) && fs_1.default.lstatSync(yarnCacheFile).isDirectory()) + return Promise.resolve(false); + // NOTE: yarn1 returns 'undefined' with rc = 0 + const enableGlobalCache = yield exports.getCommandOutput('yarn config get enableGlobalCache', workDir); + // only local cache is not managed by yarn + return enableGlobalCache === 'false'; +}); +/** + * A function to report either the repo contains at least one Yarn managed directory + * @param packageManagerInfo - used to make sure current package manager is yarn + * @return - true if there's at least one Yarn managed directory in the repo + */ +const repoHasYarn3ManagedCache = (packageManagerInfo) => __awaiter(void 0, void 0, void 0, function* () { + if (packageManagerInfo.name !== 'yarn') + return false; + const cacheDependencyPath = core.getInput('cache-dependency-path'); + const yarnDirs = cacheDependencyPath + ? yield getProjectDirectoriesFromCacheDependencyPath(cacheDependencyPath) + : ['']; + for (const dir of yarnDirs.length === 0 ? [''] : yarnDirs) { + if (yield isCacheManagedByYarn3(dir)) { + return true; + } + } + return false; +}); +exports.repoHasYarn3ManagedCache = repoHasYarn3ManagedCache; function isGhes() { const ghUrl = new URL(process.env['GITHUB_SERVER_URL'] || 'https://github.com'); return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM'; diff --git a/src/cache-restore.ts b/src/cache-restore.ts index 6ac2cc755..73d080107 100644 --- a/src/cache-restore.ts +++ b/src/cache-restore.ts @@ -8,6 +8,7 @@ import {State} from './constants'; import { getCacheDirectories, getPackageManagerInfo, + repoHasYarn3ManagedCache, PackageManagerInfo } from './cache-utils'; @@ -37,12 +38,16 @@ export const restoreCache = async ( ); } - const primaryKey = `node-cache-${platform}-${packageManager}-${fileHash}`; + const keyPrefix = `node-cache-${platform}-${packageManager}`; + const primaryKey = `${keyPrefix}-${fileHash}`; core.debug(`primary key is ${primaryKey}`); core.saveState(State.CachePrimaryKey, primaryKey); - const cacheKey = await cache.restoreCache(cachePaths, primaryKey); + const cacheKey = (await repoHasYarn3ManagedCache(packageManagerInfo)) + ? await cache.restoreCache(cachePaths, primaryKey, [keyPrefix]) + : await cache.restoreCache(cachePaths, primaryKey); + core.setOutput('cache-hit', Boolean(cacheKey)); if (!cacheKey) { diff --git a/src/cache-utils.ts b/src/cache-utils.ts index ac82c4c20..c2466f3b2 100644 --- a/src/cache-utils.ts +++ b/src/cache-utils.ts @@ -110,6 +110,20 @@ export const getPackageManagerInfo = async (packageManager: string) => { } }; +/** + * getProjectDirectoriesFromCacheDependencyPath is called twice during `restoreCache` + * - first through `getCacheDirectories` + * - second from `repoHasYarn3ManagedCache` + * + * it contains expensive IO operation and thus should be memoized + */ + +let projectDirectoriesMemoized: string[] | null = null; +/** + * unit test must reset memoized variables + */ +export const resetProjectDirectoriesMemoized = () => + (projectDirectoriesMemoized = null); /** * Expands (converts) the string input `cache-dependency-path` to list of directories that * may be project roots @@ -120,6 +134,10 @@ export const getPackageManagerInfo = async (packageManager: string) => { const getProjectDirectoriesFromCacheDependencyPath = async ( cacheDependencyPath: string ): Promise => { + if (projectDirectoriesMemoized !== null) { + return projectDirectoriesMemoized; + } + const globber = await glob.create(cacheDependencyPath); const cacheDependenciesPaths = await globber.glob(); @@ -133,6 +151,7 @@ const getProjectDirectoriesFromCacheDependencyPath = async ( `No existing directories found containing cache-dependency-path="${cacheDependencyPath}"` ); + projectDirectoriesMemoized = existingDirectories; return existingDirectories; }; @@ -202,6 +221,58 @@ export const getCacheDirectories = async ( return getCacheDirectoriesForRootProject(packageManagerInfo); }; +/** + * A function to check if the directory is a yarn project configured to manage + * obsolete dependencies in the local cache + * @param directory - a path to the folder + * @return - true if the directory's project is yarn managed + * - if there's .yarn/cache folder do not mess with the dependencies kept in the repo, return false + * - global cache is not managed by yarn @see https://yarnpkg.com/features/offline-cache, return false + * - if local cache is not explicitly enabled (not yarn3), return false + * - return true otherwise + */ +const isCacheManagedByYarn3 = async (directory: string): Promise => { + const workDir = directory || process.env.GITHUB_WORKSPACE || '.'; + + // if .yarn/cache directory exists the cache is managed by version control system + const yarnCacheFile = path.join(workDir, '.yarn', 'cache'); + if (fs.existsSync(yarnCacheFile) && fs.lstatSync(yarnCacheFile).isDirectory()) + return Promise.resolve(false); + + // NOTE: yarn1 returns 'undefined' with rc = 0 + const enableGlobalCache = await getCommandOutput( + 'yarn config get enableGlobalCache', + workDir + ); + // only local cache is not managed by yarn + return enableGlobalCache === 'false'; +}; + +/** + * A function to report either the repo contains at least one Yarn managed directory + * @param packageManagerInfo - used to make sure current package manager is yarn + * @return - true if there's at least one Yarn managed directory in the repo + */ +export const repoHasYarn3ManagedCache = async ( + packageManagerInfo: PackageManagerInfo +): Promise => { + if (packageManagerInfo.name !== 'yarn') return false; + + const cacheDependencyPath = core.getInput('cache-dependency-path'); + + const yarnDirs = cacheDependencyPath + ? await getProjectDirectoriesFromCacheDependencyPath(cacheDependencyPath) + : ['']; + + for (const dir of yarnDirs.length === 0 ? [''] : yarnDirs) { + if (await isCacheManagedByYarn3(dir)) { + return true; + } + } + + return false; +}; + export function isGhes(): boolean { const ghUrl = new URL( process.env['GITHUB_SERVER_URL'] || 'https://github.com'