Do not ivalidate the cache entirely on lock file change (#744)

* Do not ivalidate the cache entirely on yarn3 lock file change * Use cache prefix if all sub-projects are yarn managed * Rename functions & add e2e tests
actions · Jun 27, 2023 · e33196f · e33196f
1 parent c6722d3
commit e33196f
Show file tree

Hide file tree

Showing 7 changed files with 358 additions and 16 deletions.
diff --git a/.github/workflows/e2e-cache.yml b/.github/workflows/e2e-cache.yml
@@ -146,7 +146,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: prepare sub-projects
-        run: __tests__/prepare-subprojects.sh
+        run: __tests__/prepare-yarn-subprojects.sh yarn1
 
       # expect
       #  - no errors
@@ -161,3 +161,85 @@ jobs:
           cache-dependency-path: |
             **/*.lock
             yarn.lock
+
+  yarn-subprojects-berry-local:
+    name: Test yarn subprojects all locally managed
+    strategy:
+      matrix:
+        node-version: [12, 14, 16]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: prepare sub-projects
+        run: __tests__/prepare-yarn-subprojects.sh
+
+      # expect
+      #  - no errors
+      #  - log
+      #    ##[info]All dependencies are managed locally by yarn3, the previous cache can be used
+      #    ##[debug]["node-cache-Linux-yarn-401024703386272f1a950c9f014cbb1bb79a7a5b6e1fb00e8b90d06734af41ee","node-cache-Linux-yarn"]
+      - name: Setup Node
+        uses: ./
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'yarn'
+          cache-dependency-path: |
+            sub2/*.lock
+            sub3/*.lock
+
+  yarn-subprojects-berry-global:
+    name: Test yarn subprojects some locally managed
+    strategy:
+      matrix:
+        node-version: [12, 14, 16]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: prepare sub-projects
+        run: __tests__/prepare-yarn-subprojects.sh global
+
+      # expect
+      #  - no errors
+      #  - log must
+      #    ##[debug]"/home/runner/work/setup-node-test/setup-node-test/sub2" dependencies are managed by yarn 3 locally
+      #    ##[debug]"/home/runner/work/setup-node-test/setup-node-test/sub3" dependencies are not managed by yarn 3 locally
+      - name: Setup Node
+        uses: ./
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'yarn'
+          cache-dependency-path: |
+            sub2/*.lock
+            sub3/*.lock
+
+  yarn-subprojects-berry-git:
+    name: Test yarn subprojects managed by git
+    strategy:
+      matrix:
+        node-version: [12, 14, 16]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: prepare sub-projects
+        run: /bin/bash __tests__/prepare-yarn-subprojects.sh keepcache
+
+      # expect
+      #  - no errors
+      #  - log
+      #    [debug]"/home/runner/work/setup-node-test/setup-node-test/sub2" has .yarn/cache - dependencies are kept in the repository
+      #    [debug]"/home/runner/work/setup-node-test/setup-node-test/sub3" has .yarn/cache - dependencies are kept in the repository
+      #    [debug]["node-cache-Linux-yarn-401024703386272f1a950c9f014cbb1bb79a7a5b6e1fb00e8b90d06734af41ee"]
+      - name: Setup Node
+        uses: ./
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'yarn'
+          cache-dependency-path: |
+            sub2/*.lock
+            sub3/*.lock
diff --git a/__tests__/cache-utils.test.ts b/__tests__/cache-utils.test.ts
@@ -6,7 +6,8 @@ import {
   PackageManagerInfo,
   isCacheFeatureAvailable,
   supportedPackageManagers,
-  getCommandOutput
+  getCommandOutput,
+  resetProjectDirectoriesMemoized
 } from '../src/cache-utils';
 import fs from 'fs';
 import * as cacheUtils from '../src/cache-utils';
@@ -103,6 +104,8 @@ describe('cache-utils', () => {
         (pattern: string): Promise<Globber> =>
           MockGlobber.create(['/foo', '/bar'])
       );
+
+      resetProjectDirectoriesMemoized();
     });
 
     afterEach(() => {

diff --git a/__tests__/prepare-subprojects.sh → __tests__/prepare-yarn-subprojects.sh b/__tests__/prepare-subprojects.sh → __tests__/prepare-yarn-subprojects.sh
@@ -32,10 +32,20 @@ cat <<EOT >package.json
 EOT
 yarn set version 3.5.1
 yarn install
+if [ x$1 = 'xglobal' ];then
+  echo enableGlobalCache
+  echo 'enableGlobalCache: true' >> .yarnrc.yml
+fi
 
-echo "create yarn1 project in the root"
 cd ..
-cat <<EOT >package.json
+if [ x$1 != 'xkeepcache' -a x$2 != 'xkeepcache' ]; then
+  rm -rf sub2/.yarn/cache
+  rm -rf sub3/.yarn/cache
+fi
+
+if [ x$1 = 'xyarn1' ];then
+  echo "create yarn1 project in the root"
+  cat <<EOT >package.json
 {
   "name": "subproject",
   "dependencies": {
@@ -44,5 +54,6 @@ cat <<EOT >package.json
   }
 }
 EOT
-yarn set version 1.22.19
-yarn install
+  yarn set version 1.22.19
+  yarn install
+fi
diff --git a/dist/cache-save/index.js b/dist/cache-save/index.js
@@ -60434,7 +60434,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", ({ value: true }));
-exports.isCacheFeatureAvailable = exports.isGhes = exports.getCacheDirectories = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0;
+exports.isCacheFeatureAvailable = exports.isGhes = exports.repoHasYarnBerryManagedDependencies = exports.getCacheDirectories = exports.resetProjectDirectoriesMemoized = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0;
 const core = __importStar(__nccwpck_require__(2186));
 const exec = __importStar(__nccwpck_require__(1514));
 const cache = __importStar(__nccwpck_require__(7799));
@@ -60503,6 +60503,19 @@ const getPackageManagerInfo = (packageManager) => __awaiter(void 0, void 0, void
     }
 });
 exports.getPackageManagerInfo = getPackageManagerInfo;
+/**
+ * getProjectDirectoriesFromCacheDependencyPath is called twice during `restoreCache`
+ *  - first through `getCacheDirectories`
+ *  - second from `repoHasYarn3ManagedCache`
+ *
+ *  it contains expensive IO operation and thus should be memoized
+ */
+let projectDirectoriesMemoized = null;
+/**
+ * unit test must reset memoized variables
+ */
+const resetProjectDirectoriesMemoized = () => (projectDirectoriesMemoized = null);
+exports.resetProjectDirectoriesMemoized = resetProjectDirectoriesMemoized;
 /**
  * Expands (converts) the string input `cache-dependency-path` to list of directories that
  * may be project roots
@@ -60511,6 +60524,9 @@ exports.getPackageManagerInfo = getPackageManagerInfo;
  * @return list of directories and possible
  */
 const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
+    if (projectDirectoriesMemoized !== null) {
+        return projectDirectoriesMemoized;
+    }
     const globber = yield glob.create(cacheDependencyPath);
     const cacheDependenciesPaths = yield globber.glob();
     const existingDirectories = cacheDependenciesPaths
@@ -60519,6 +60535,7 @@ const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __
         .filter(directory => fs_1.default.lstatSync(directory).isDirectory());
     if (!existingDirectories.length)
         core.warning(`No existing directories found containing cache-dependency-path="${cacheDependencyPath}"`);
+    projectDirectoriesMemoized = existingDirectories;
     return existingDirectories;
 });
 /**
@@ -60531,7 +60548,7 @@ const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __
 const getCacheDirectoriesFromCacheDependencyPath = (packageManagerInfo, cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
     const projectDirectories = yield getProjectDirectoriesFromCacheDependencyPath(cacheDependencyPath);
     const cacheFoldersPaths = yield Promise.all(projectDirectories.map((projectDirectory) => __awaiter(void 0, void 0, void 0, function* () {
-        const cacheFolderPath = packageManagerInfo.getCacheFolderPath(projectDirectory);
+        const cacheFolderPath = yield packageManagerInfo.getCacheFolderPath(projectDirectory);
         core.debug(`${packageManagerInfo.name}'s cache folder "${cacheFolderPath}" configured for the directory "${projectDirectory}"`);
         return cacheFolderPath;
     })));
@@ -60565,6 +60582,56 @@ const getCacheDirectories = (packageManagerInfo, cacheDependencyPath) => __await
     return getCacheDirectoriesForRootProject(packageManagerInfo);
 });
 exports.getCacheDirectories = getCacheDirectories;
+/**
+ * A function to check if the directory is a yarn project configured to manage
+ * obsolete dependencies in the local cache
+ * @param directory - a path to the folder
+ * @return - true if the directory's project is yarn managed
+ *  - if there's .yarn/cache folder do not mess with the dependencies kept in the repo, return false
+ *  - global cache is not managed by yarn @see https://yarnpkg.com/features/offline-cache, return false
+ *  - if local cache is not explicitly enabled (not yarn3), return false
+ *  - return true otherwise
+ */
+const projectHasYarnBerryManagedDependencies = (directory) => __awaiter(void 0, void 0, void 0, function* () {
+    const workDir = directory || process.env.GITHUB_WORKSPACE || '.';
+    core.debug(`check if "${workDir}" has locally managed yarn3 dependencies`);
+    // if .yarn/cache directory exists the cache is managed by version control system
+    const yarnCacheFile = path_1.default.join(workDir, '.yarn', 'cache');
+    if (fs_1.default.existsSync(yarnCacheFile) &&
+        fs_1.default.lstatSync(yarnCacheFile).isDirectory()) {
+        core.debug(`"${workDir}" has .yarn/cache - dependencies are kept in the repository`);
+        return Promise.resolve(false);
+    }
+    // NOTE: yarn1 returns 'undefined' with return code = 0
+    const enableGlobalCache = yield exports.getCommandOutput('yarn config get enableGlobalCache', workDir);
+    // only local cache is not managed by yarn
+    const managed = enableGlobalCache.includes('false');
+    if (managed) {
+        core.debug(`"${workDir}" dependencies are managed by yarn 3 locally`);
+        return true;
+    }
+    else {
+        core.debug(`"${workDir}" dependencies are not managed by yarn 3 locally`);
+        return false;
+    }
+});
+/**
+ * A function to report the repo contains Yarn managed projects
+ * @param packageManagerInfo - used to make sure current package manager is yarn
+ * @param cacheDependencyPath - either a single string or multiline string with possible glob patterns
+ *                              expected to be the result of `core.getInput('cache-dependency-path')`
+ * @return - true if all project directories configured to be Yarn managed
+ */
+const repoHasYarnBerryManagedDependencies = (packageManagerInfo, cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
+    if (packageManagerInfo.name !== 'yarn')
+        return false;
+    const yarnDirs = cacheDependencyPath
+        ? yield getProjectDirectoriesFromCacheDependencyPath(cacheDependencyPath)
+        : [''];
+    const isManagedList = yield Promise.all(yarnDirs.map(projectHasYarnBerryManagedDependencies));
+    return isManagedList.every(Boolean);
+});
+exports.repoHasYarnBerryManagedDependencies = repoHasYarnBerryManagedDependencies;
 function isGhes() {
     const ghUrl = new URL(process.env['GITHUB_SERVER_URL'] || 'https://github.com');
     return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM';

diff --git a/dist/setup/index.js b/dist/setup/index.js
@@ -71153,10 +71153,19 @@ const restoreCache = (packageManager, cacheDependencyPath) => __awaiter(void 0,
     if (!fileHash) {
         throw new Error('Some specified paths were not resolved, unable to cache dependencies.');
     }
-    const primaryKey = `node-cache-${platform}-${packageManager}-${fileHash}`;
+    const keyPrefix = `node-cache-${platform}-${packageManager}`;
+    const primaryKey = `${keyPrefix}-${fileHash}`;
     core.debug(`primary key is ${primaryKey}`);
     core.saveState(constants_1.State.CachePrimaryKey, primaryKey);
-    const cacheKey = yield cache.restoreCache(cachePaths, primaryKey);
+    const isManagedByYarnBerry = yield cache_utils_1.repoHasYarnBerryManagedDependencies(packageManagerInfo, cacheDependencyPath);
+    let cacheKey;
+    if (isManagedByYarnBerry) {
+        core.info('All dependencies are managed locally by yarn3, the previous cache can be used');
+        cacheKey = yield cache.restoreCache(cachePaths, primaryKey, [keyPrefix]);
+    }
+    else {
+        cacheKey = yield cache.restoreCache(cachePaths, primaryKey);
+    }
     core.setOutput('cache-hit', Boolean(cacheKey));
     if (!cacheKey) {
         core.info(`${packageManager} cache is not found`);
@@ -71217,7 +71226,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", ({ value: true }));
-exports.isCacheFeatureAvailable = exports.isGhes = exports.getCacheDirectories = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0;
+exports.isCacheFeatureAvailable = exports.isGhes = exports.repoHasYarnBerryManagedDependencies = exports.getCacheDirectories = exports.resetProjectDirectoriesMemoized = exports.getPackageManagerInfo = exports.getCommandOutputNotEmpty = exports.getCommandOutput = exports.supportedPackageManagers = void 0;
 const core = __importStar(__nccwpck_require__(2186));
 const exec = __importStar(__nccwpck_require__(1514));
 const cache = __importStar(__nccwpck_require__(7799));
@@ -71286,6 +71295,19 @@ const getPackageManagerInfo = (packageManager) => __awaiter(void 0, void 0, void
     }
 });
 exports.getPackageManagerInfo = getPackageManagerInfo;
+/**
+ * getProjectDirectoriesFromCacheDependencyPath is called twice during `restoreCache`
+ *  - first through `getCacheDirectories`
+ *  - second from `repoHasYarn3ManagedCache`
+ *
+ *  it contains expensive IO operation and thus should be memoized
+ */
+let projectDirectoriesMemoized = null;
+/**
+ * unit test must reset memoized variables
+ */
+const resetProjectDirectoriesMemoized = () => (projectDirectoriesMemoized = null);
+exports.resetProjectDirectoriesMemoized = resetProjectDirectoriesMemoized;
 /**
  * Expands (converts) the string input `cache-dependency-path` to list of directories that
  * may be project roots
@@ -71294,6 +71316,9 @@ exports.getPackageManagerInfo = getPackageManagerInfo;
  * @return list of directories and possible
  */
 const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
+    if (projectDirectoriesMemoized !== null) {
+        return projectDirectoriesMemoized;
+    }
     const globber = yield glob.create(cacheDependencyPath);
     const cacheDependenciesPaths = yield globber.glob();
     const existingDirectories = cacheDependenciesPaths
@@ -71302,6 +71327,7 @@ const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __
         .filter(directory => fs_1.default.lstatSync(directory).isDirectory());
     if (!existingDirectories.length)
         core.warning(`No existing directories found containing cache-dependency-path="${cacheDependencyPath}"`);
+    projectDirectoriesMemoized = existingDirectories;
     return existingDirectories;
 });
 /**
@@ -71314,7 +71340,7 @@ const getProjectDirectoriesFromCacheDependencyPath = (cacheDependencyPath) => __
 const getCacheDirectoriesFromCacheDependencyPath = (packageManagerInfo, cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
     const projectDirectories = yield getProjectDirectoriesFromCacheDependencyPath(cacheDependencyPath);
     const cacheFoldersPaths = yield Promise.all(projectDirectories.map((projectDirectory) => __awaiter(void 0, void 0, void 0, function* () {
-        const cacheFolderPath = packageManagerInfo.getCacheFolderPath(projectDirectory);
+        const cacheFolderPath = yield packageManagerInfo.getCacheFolderPath(projectDirectory);
         core.debug(`${packageManagerInfo.name}'s cache folder "${cacheFolderPath}" configured for the directory "${projectDirectory}"`);
         return cacheFolderPath;
     })));
@@ -71348,6 +71374,56 @@ const getCacheDirectories = (packageManagerInfo, cacheDependencyPath) => __await
     return getCacheDirectoriesForRootProject(packageManagerInfo);
 });
 exports.getCacheDirectories = getCacheDirectories;
+/**
+ * A function to check if the directory is a yarn project configured to manage
+ * obsolete dependencies in the local cache
+ * @param directory - a path to the folder
+ * @return - true if the directory's project is yarn managed
+ *  - if there's .yarn/cache folder do not mess with the dependencies kept in the repo, return false
+ *  - global cache is not managed by yarn @see https://yarnpkg.com/features/offline-cache, return false
+ *  - if local cache is not explicitly enabled (not yarn3), return false
+ *  - return true otherwise
+ */
+const projectHasYarnBerryManagedDependencies = (directory) => __awaiter(void 0, void 0, void 0, function* () {
+    const workDir = directory || process.env.GITHUB_WORKSPACE || '.';
+    core.debug(`check if "${workDir}" has locally managed yarn3 dependencies`);
+    // if .yarn/cache directory exists the cache is managed by version control system
+    const yarnCacheFile = path_1.default.join(workDir, '.yarn', 'cache');
+    if (fs_1.default.existsSync(yarnCacheFile) &&
+        fs_1.default.lstatSync(yarnCacheFile).isDirectory()) {
+        core.debug(`"${workDir}" has .yarn/cache - dependencies are kept in the repository`);
+        return Promise.resolve(false);
+    }
+    // NOTE: yarn1 returns 'undefined' with return code = 0
+    const enableGlobalCache = yield exports.getCommandOutput('yarn config get enableGlobalCache', workDir);
+    // only local cache is not managed by yarn
+    const managed = enableGlobalCache.includes('false');
+    if (managed) {
+        core.debug(`"${workDir}" dependencies are managed by yarn 3 locally`);
+        return true;
+    }
+    else {
+        core.debug(`"${workDir}" dependencies are not managed by yarn 3 locally`);
+        return false;
+    }
+});
+/**
+ * A function to report the repo contains Yarn managed projects
+ * @param packageManagerInfo - used to make sure current package manager is yarn
+ * @param cacheDependencyPath - either a single string or multiline string with possible glob patterns
+ *                              expected to be the result of `core.getInput('cache-dependency-path')`
+ * @return - true if all project directories configured to be Yarn managed
+ */
+const repoHasYarnBerryManagedDependencies = (packageManagerInfo, cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
+    if (packageManagerInfo.name !== 'yarn')
+        return false;
+    const yarnDirs = cacheDependencyPath
+        ? yield getProjectDirectoriesFromCacheDependencyPath(cacheDependencyPath)
+        : [''];
+    const isManagedList = yield Promise.all(yarnDirs.map(projectHasYarnBerryManagedDependencies));
+    return isManagedList.every(Boolean);
+});
+exports.repoHasYarnBerryManagedDependencies = repoHasYarnBerryManagedDependencies;
 function isGhes() {
     const ghUrl = new URL(process.env['GITHUB_SERVER_URL'] || 'https://github.com');
     return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM';