Skip to content

Commit

Permalink
Do not ivalidate the cache entirely on yarn3 lock file change
Browse files Browse the repository at this point in the history
  • Loading branch information
@dsame
dsame committed Jun 22, 2023
1 parent 8170e22 commit 342f9b8
Show file tree
Hide file tree
Showing 5 changed files with 205 additions and 7 deletions.
5 changes: 4 additions & 1 deletion __tests__/cache-utils.test.ts
View file
Expand Up @@ -6,7 +6,8 @@ import {
PackageManagerInfo,
isCacheFeatureAvailable,
supportedPackageManagers,
getCommandOutput
getCommandOutput,
resetProjectDirectoriesMemoized
} from '../src/cache-utils';
import fs from 'fs';
import * as cacheUtils from '../src/cache-utils';
Expand Down Expand Up @@ -103,6 +104,8 @@ describe('cache-utils', () => {
(pattern: string): Promise<Globber> =>
MockGlobber.create(['/foo', '/bar'])
);

resetProjectDirectoriesMemoized();
});

afterEach(() => {
Expand Down
60 changes: 59 additions & 1 deletion dist/cache-save/index.js
View file
Expand Up @@ -60434,7 +60434,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.isCacheFeatureAvailable = exports.isGhes = exports.getCacheDirectories = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0;
exports.isCacheFeatureAvailable = exports.isGhes = exports.repoHasYarn3ManagedCache = exports.getCacheDirectories = exports.resetProjectDirectoriesMemoized = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0;
const core = __importStar(__nccwpck_require__(2186));
const exec = __importStar(__nccwpck_require__(1514));
const cache = __importStar(__nccwpck_require__(7799));
Expand Down Expand Up @@ -60503,6 +60503,19 @@ const getPackageManagerInfo = (packageManager) => __awaiter(void 0, void 0, void
}
});
exports.getPackageManagerInfo = getPackageManagerInfo;
/**
* getProjectDirectoriesFromCacheDependencyPath is called twice during `restoreCache`
* - first through `getCacheDirectories`
* - second from `repoHasYarn3ManagedCache`
*
* it contains expensive IO operation and thus should be memoized
*/
let projectDirectoriesMemoized = null;
/**
* unit test must reset memoized variables
*/
const resetProjectDirectoriesMemoized = () => (projectDirectoriesMemoized = null);
exports.resetProjectDirectoriesMemoized = resetProjectDirectoriesMemoized;
/**
* Expands (converts) the string input `cache-dependency-path` to list of directories that
* may be project roots
Expand All @@ -60511,6 +60524,9 @@ exports.getPackageManagerInfo = getPackageManagerInfo;
* @return list of directories and possible
*/
const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
if (projectDirectoriesMemoized !== null) {
return projectDirectoriesMemoized;
}
const globber = yield glob.create(cacheDependencyPath);
const cacheDependenciesPaths = yield globber.glob();
const existingDirectories = cacheDependenciesPaths
Expand All @@ -60519,6 +60535,7 @@ const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __
.filter(directory => fs_1.default.lstatSync(directory).isDirectory());
if (!existingDirectories.length)
core.warning(`No existing directories found containing cache-dependency-path="${cacheDependencyPath}"`);
projectDirectoriesMemoized = existingDirectories;
return existingDirectories;
});
/**
Expand Down Expand Up @@ -60565,6 +60582,47 @@ const getCacheDirectories = (packageManagerInfo, cacheDependencyPath) => __await
return getCacheDirectoriesForRootProject(packageManagerInfo);
});
exports.getCacheDirectories = getCacheDirectories;
/**
* A function to check if the directory is a yarn project configured to manage
* obsolete dependencies in the local cache
* @param directory - a path to the folder
* @return - true if the directory's project is yarn managed
* - if there's .yarn/cache folder do not mess with the dependencies kept in the repo, return false
* - global cache is not managed by yarn @see https://yarnpkg.com/features/offline-cache, return false
* - if local cache is not explicitly enabled (not yarn3), return false
* - return true otherwise
*/
const isCacheManagedByYarn3 = (directory) => __awaiter(void 0, void 0, void 0, function* () {
const workDir = directory || process.env.GITHUB_WORKSPACE || '.';
// if .yarn/cache directory exists the cache is managed by version control system
const yarnCacheFile = path_1.default.join(workDir, '.yarn', 'cache');
if (fs_1.default.existsSync(yarnCacheFile) && fs_1.default.lstatSync(yarnCacheFile).isDirectory())
return Promise.resolve(false);
// NOTE: yarn1 returns 'undefined' with rc = 0
const enableGlobalCache = yield exports.getCommandOutput('yarn config get enableGlobalCache', workDir);
// only local cache is not managed by yarn
return enableGlobalCache === 'false';
});
/**
* A function to report either the repo contains at least one Yarn managed directory
* @param packageManagerInfo - used to make sure current package manager is yarn
* @return - true if there's at least one Yarn managed directory in the repo
*/
const repoHasYarn3ManagedCache = (packageManagerInfo) => __awaiter(void 0, void 0, void 0, function* () {
if (packageManagerInfo.name !== 'yarn')
return false;
const cacheDependencyPath = core.getInput('cache-dependency-path');
const yarnDirs = cacheDependencyPath
? yield getProjectDirectoriesFromCacheDependencyPath(cacheDependencyPath)
: [''];
for (const dir of yarnDirs.length === 0 ? [''] : yarnDirs) {
if (yield isCacheManagedByYarn3(dir)) {
return true;
}
}
return false;
});
exports.repoHasYarn3ManagedCache = repoHasYarn3ManagedCache;
function isGhes() {
const ghUrl = new URL(process.env['GITHUB_SERVER_URL'] || 'https://github.com');
return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM';
Expand Down
67 changes: 64 additions & 3 deletions dist/setup/index.js
View file
Expand Up @@ -71153,10 +71153,13 @@ const restoreCache = (packageManager, cacheDependencyPath) => __awaiter(void 0,
if (!fileHash) {
throw new Error('Some specified paths were not resolved, unable to cache dependencies.');
}
const primaryKey = `node-cache-${platform}-${packageManager}-${fileHash}`;
const keyPrefix = `node-cache-${platform}-${packageManager}`;
const primaryKey = `${keyPrefix}-${fileHash}`;
core.debug(`primary key is ${primaryKey}`);
core.saveState(constants_1.State.CachePrimaryKey, primaryKey);
const cacheKey = yield cache.restoreCache(cachePaths, primaryKey);
const cacheKey = (yield cache_utils_1.repoHasYarn3ManagedCache(packageManagerInfo))
? yield cache.restoreCache(cachePaths, primaryKey, [keyPrefix])
: yield cache.restoreCache(cachePaths, primaryKey);
core.setOutput('cache-hit', Boolean(cacheKey));
if (!cacheKey) {
core.info(`${packageManager} cache is not found`);
Expand Down Expand Up @@ -71217,7 +71220,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.isCacheFeatureAvailable = exports.isGhes = exports.getCacheDirectories = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0;
exports.isCacheFeatureAvailable = exports.isGhes = exports.repoHasYarn3ManagedCache = exports.getCacheDirectories = exports.resetProjectDirectoriesMemoized = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0;
const core = __importStar(__nccwpck_require__(2186));
const exec = __importStar(__nccwpck_require__(1514));
const cache = __importStar(__nccwpck_require__(7799));
Expand Down Expand Up @@ -71286,6 +71289,19 @@ const getPackageManagerInfo = (packageManager) => __awaiter(void 0, void 0, void
}
});
exports.getPackageManagerInfo = getPackageManagerInfo;
/**
* getProjectDirectoriesFromCacheDependencyPath is called twice during `restoreCache`
* - first through `getCacheDirectories`
* - second from `repoHasYarn3ManagedCache`
*
* it contains expensive IO operation and thus should be memoized
*/
let projectDirectoriesMemoized = null;
/**
* unit test must reset memoized variables
*/
const resetProjectDirectoriesMemoized = () => (projectDirectoriesMemoized = null);
exports.resetProjectDirectoriesMemoized = resetProjectDirectoriesMemoized;
/**
* Expands (converts) the string input `cache-dependency-path` to list of directories that
* may be project roots
Expand All @@ -71294,6 +71310,9 @@ exports.getPackageManagerInfo = getPackageManagerInfo;
* @return list of directories and possible
*/
const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
if (projectDirectoriesMemoized !== null) {
return projectDirectoriesMemoized;
}
const globber = yield glob.create(cacheDependencyPath);
const cacheDependenciesPaths = yield globber.glob();
const existingDirectories = cacheDependenciesPaths
Expand All @@ -71302,6 +71321,7 @@ const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __
.filter(directory => fs_1.default.lstatSync(directory).isDirectory());
if (!existingDirectories.length)
core.warning(`No existing directories found containing cache-dependency-path="${cacheDependencyPath}"`);
projectDirectoriesMemoized = existingDirectories;
return existingDirectories;
});
/**
Expand Down Expand Up @@ -71348,6 +71368,47 @@ const getCacheDirectories = (packageManagerInfo, cacheDependencyPath) => __await
return getCacheDirectoriesForRootProject(packageManagerInfo);
});
exports.getCacheDirectories = getCacheDirectories;
/**
* A function to check if the directory is a yarn project configured to manage
* obsolete dependencies in the local cache
* @param directory - a path to the folder
* @return - true if the directory's project is yarn managed
* - if there's .yarn/cache folder do not mess with the dependencies kept in the repo, return false
* - global cache is not managed by yarn @see https://yarnpkg.com/features/offline-cache, return false
* - if local cache is not explicitly enabled (not yarn3), return false
* - return true otherwise
*/
const isCacheManagedByYarn3 = (directory) => __awaiter(void 0, void 0, void 0, function* () {
const workDir = directory || process.env.GITHUB_WORKSPACE || '.';
// if .yarn/cache directory exists the cache is managed by version control system
const yarnCacheFile = path_1.default.join(workDir, '.yarn', 'cache');
if (fs_1.default.existsSync(yarnCacheFile) && fs_1.default.lstatSync(yarnCacheFile).isDirectory())
return Promise.resolve(false);
// NOTE: yarn1 returns 'undefined' with rc = 0
const enableGlobalCache = yield exports.getCommandOutput('yarn config get enableGlobalCache', workDir);
// only local cache is not managed by yarn
return enableGlobalCache === 'false';
});
/**
* A function to report either the repo contains at least one Yarn managed directory
* @param packageManagerInfo - used to make sure current package manager is yarn
* @return - true if there's at least one Yarn managed directory in the repo
*/
const repoHasYarn3ManagedCache = (packageManagerInfo) => __awaiter(void 0, void 0, void 0, function* () {
if (packageManagerInfo.name !== 'yarn')
return false;
const cacheDependencyPath = core.getInput('cache-dependency-path');
const yarnDirs = cacheDependencyPath
? yield getProjectDirectoriesFromCacheDependencyPath(cacheDependencyPath)
: [''];
for (const dir of yarnDirs.length === 0 ? [''] : yarnDirs) {
if (yield isCacheManagedByYarn3(dir)) {
return true;
}
}
return false;
});
exports.repoHasYarn3ManagedCache = repoHasYarn3ManagedCache;
function isGhes() {
const ghUrl = new URL(process.env['GITHUB_SERVER_URL'] || 'https://github.com');
return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM';
Expand Down
9 changes: 7 additions & 2 deletions src/cache-restore.ts
View file
Expand Up @@ -8,6 +8,7 @@ import {State} from './constants';
import {
getCacheDirectories,
getPackageManagerInfo,
repoHasYarn3ManagedCache,
PackageManagerInfo
} from './cache-utils';

Expand Down Expand Up @@ -37,12 +38,16 @@ export const restoreCache = async (
);
}

const primaryKey = `node-cache-${platform}-${packageManager}-${fileHash}`;
const keyPrefix = `node-cache-${platform}-${packageManager}`;
const primaryKey = `${keyPrefix}-${fileHash}`;
core.debug(`primary key is ${primaryKey}`);

core.saveState(State.CachePrimaryKey, primaryKey);

const cacheKey = await cache.restoreCache(cachePaths, primaryKey);
const cacheKey = (await repoHasYarn3ManagedCache(packageManagerInfo))
? await cache.restoreCache(cachePaths, primaryKey, [keyPrefix])
: await cache.restoreCache(cachePaths, primaryKey);

core.setOutput('cache-hit', Boolean(cacheKey));

if (!cacheKey) {
Expand Down
71 changes: 71 additions & 0 deletions src/cache-utils.ts
View file
Expand Up @@ -110,6 +110,20 @@ export const getPackageManagerInfo = async (packageManager: string) => {
}
};

/**
* getProjectDirectoriesFromCacheDependencyPath is called twice during `restoreCache`
* - first through `getCacheDirectories`
* - second from `repoHasYarn3ManagedCache`
*
* it contains expensive IO operation and thus should be memoized
*/

let projectDirectoriesMemoized: string[] | null = null;
/**
* unit test must reset memoized variables
*/
export const resetProjectDirectoriesMemoized = () =>
(projectDirectoriesMemoized = null);
/**
* Expands (converts) the string input `cache-dependency-path` to list of directories that
* may be project roots
Expand All @@ -120,6 +134,10 @@ export const getPackageManagerInfo = async (packageManager: string) => {
const getProjectDirectoriesFromCacheDependencyPath = async (
cacheDependencyPath: string
): Promise<string[]> => {
if (projectDirectoriesMemoized !== null) {
return projectDirectoriesMemoized;
}

const globber = await glob.create(cacheDependencyPath);
const cacheDependenciesPaths = await globber.glob();

Expand All @@ -133,6 +151,7 @@ const getProjectDirectoriesFromCacheDependencyPath = async (
`No existing directories found containing cache-dependency-path="${cacheDependencyPath}"`
);

projectDirectoriesMemoized = existingDirectories;
return existingDirectories;
};

Expand Down Expand Up @@ -202,6 +221,58 @@ export const getCacheDirectories = async (
return getCacheDirectoriesForRootProject(packageManagerInfo);
};

/**
* A function to check if the directory is a yarn project configured to manage
* obsolete dependencies in the local cache
* @param directory - a path to the folder
* @return - true if the directory's project is yarn managed
* - if there's .yarn/cache folder do not mess with the dependencies kept in the repo, return false
* - global cache is not managed by yarn @see https://yarnpkg.com/features/offline-cache, return false
* - if local cache is not explicitly enabled (not yarn3), return false
* - return true otherwise
*/
const isCacheManagedByYarn3 = async (directory: string): Promise<boolean> => {
const workDir = directory || process.env.GITHUB_WORKSPACE || '.';

// if .yarn/cache directory exists the cache is managed by version control system
const yarnCacheFile = path.join(workDir, '.yarn', 'cache');
if (fs.existsSync(yarnCacheFile) && fs.lstatSync(yarnCacheFile).isDirectory())
return Promise.resolve(false);

// NOTE: yarn1 returns 'undefined' with rc = 0
const enableGlobalCache = await getCommandOutput(
'yarn config get enableGlobalCache',
workDir
);
// only local cache is not managed by yarn
return enableGlobalCache === 'false';
};

/**
* A function to report either the repo contains at least one Yarn managed directory
* @param packageManagerInfo - used to make sure current package manager is yarn
* @return - true if there's at least one Yarn managed directory in the repo
*/
export const repoHasYarn3ManagedCache = async (
packageManagerInfo: PackageManagerInfo
): Promise<boolean> => {
if (packageManagerInfo.name !== 'yarn') return false;

const cacheDependencyPath = core.getInput('cache-dependency-path');

const yarnDirs = cacheDependencyPath
? await getProjectDirectoriesFromCacheDependencyPath(cacheDependencyPath)
: [''];

for (const dir of yarnDirs.length === 0 ? [''] : yarnDirs) {
if (await isCacheManagedByYarn3(dir)) {
return true;
}
}

return false;
};

export function isGhes(): boolean {
const ghUrl = new URL(
process.env['GITHUB_SERVER_URL'] || 'https://github.com'
Expand Down

0 comments on commit 342f9b8

Please sign in to comment.