From cf004f0dc387df9d54b2a4aac1ad761851b90d0f Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Fri, 15 Sep 2023 22:43:13 +0200 Subject: [PATCH] Sources modified --- __tests__/cache-restore.test.ts | 6 +- action.yml | 21 ++++- package-lock.json | 20 ++++- package.json | 2 + src/cache-restore.ts | 60 +++++++++++-- src/cache-save.ts | 61 +++++++++++-- src/cache-utils.ts | 149 ++++++++++++++++++++------------ src/constants.ts | 9 +- src/main.ts | 34 ++++++-- src/package-managers.ts | 2 +- 10 files changed, 273 insertions(+), 91 deletions(-) diff --git a/__tests__/cache-restore.test.ts b/__tests__/cache-restore.test.ts index 07dc97aef..f3dbabe1c 100644 --- a/__tests__/cache-restore.test.ts +++ b/__tests__/cache-restore.test.ts @@ -41,7 +41,7 @@ describe('restoreCache', () => { //Act + Assert await expect(async () => { - await cacheRestore.restoreCache( + await cacheRestore.restoreModCache( versionSpec, packageManager, cacheDependencyPath @@ -66,7 +66,7 @@ describe('restoreCache', () => { }); //Act + Assert - await cacheRestore.restoreCache( + await cacheRestore.restoreModCache( versionSpec, packageManager, cacheDependencyPath @@ -89,7 +89,7 @@ describe('restoreCache', () => { }); //Act + Assert - await cacheRestore.restoreCache( + await cacheRestore.restoreModCache( versionSpec, packageManager, cacheDependencyPath diff --git a/action.yml b/action.yml index 9946e476e..5374c316a 100644 --- a/action.yml +++ b/action.yml @@ -13,17 +13,32 @@ inputs: description: Used to pull Go distributions from go-versions. Since there's a default, this is typically not supplied by the user. When running this action on github.com, the default value is sufficient. When running on GHES, you can pass a personal access token for github.com if you are experiencing rate limiting. default: ${{ github.server_url == 'https://github.com' && github.token || '' }} cache: - description: Used to specify whether caching is needed. Set to true, if you'd like to enable caching. + description: Used to specify whether caching is needed. Set to true, if you'd like to enable caching for both modules and intermediate build results. + default: true + cache-mod: + description: Used to specify whether modules caching is needed. Set to false, if you've found it increase the overall build time. + default: true + cache-build: + description: Used to specify whether caching of intermediate build files is needed. Set to false, if you've found it increase the overall build time. default: true cache-dependency-path: - description: 'Used to specify the path to a dependency file - go.sum' + description: 'Used to specify the path or glob pattern to a file(s) the caching of modules depends on, default: go.sum' + cache-build-path: + description: 'Used to specify the path or glob pattern to a file(s) that affect the caching of intermediate build results, default: **/*.go' + cache-id: + description: 'Used to modify cache ID if the parallel workflows must not share the same cache, default: none' + cache-lookup-only: + description: 'Use the cache created by another workflow, but do not update it' + default: false architecture: description: 'Target architecture for Go to use. Examples: x86, x64. Will use system architecture by default.' outputs: go-version: description: 'The installed Go version. Useful when given a version range as input.' cache-hit: - description: 'A boolean value to indicate if a cache was hit' + description: 'A boolean value to indicate if a modules cache was hit' + cache-build-hit: + description: 'A boolean value to indicate if a intermediate build results cache was hit' runs: using: 'node20' main: 'dist/setup/index.js' diff --git a/package-lock.json b/package-lock.json index 56773104d..73629dd63 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,10 +16,12 @@ "@actions/http-client": "^2.0.1", "@actions/io": "^1.0.2", "@actions/tool-cache": "^1.5.5", + "lodash.memoize": "^4.1.2", "semver": "^6.3.1" }, "devDependencies": { "@types/jest": "^27.0.2", + "@types/lodash": "^4.14.198", "@types/node": "^16.11.25", "@types/semver": "^6.0.0", "@typescript-eslint/eslint-plugin": "^5.54.0", @@ -1464,6 +1466,12 @@ "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", "dev": true }, + "node_modules/@types/lodash": { + "version": "4.14.198", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.198.tgz", + "integrity": "sha512-trNJ/vtMZYMLhfN45uLq4ShQSw0/S7xCTLLVM+WM1rmFpba/VS42jVUgaO3w/NOLiWR/09lnYk0yMaA/atdIsg==", + "dev": true + }, "node_modules/@types/node": { "version": "16.11.26", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.26.tgz", @@ -4721,8 +4729,7 @@ "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", - "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", - "dev": true + "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==" }, "node_modules/lodash.merge": { "version": "4.6.2", @@ -7451,6 +7458,12 @@ "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", "dev": true }, + "@types/lodash": { + "version": "4.14.198", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.198.tgz", + "integrity": "sha512-trNJ/vtMZYMLhfN45uLq4ShQSw0/S7xCTLLVM+WM1rmFpba/VS42jVUgaO3w/NOLiWR/09lnYk0yMaA/atdIsg==", + "dev": true + }, "@types/node": { "version": "16.11.26", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.26.tgz", @@ -9854,8 +9867,7 @@ "lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", - "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", - "dev": true + "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==" }, "lodash.merge": { "version": "4.6.2", diff --git a/package.json b/package.json index 71fdcc68c..89fa7c25c 100644 --- a/package.json +++ b/package.json @@ -32,10 +32,12 @@ "@actions/http-client": "^2.0.1", "@actions/io": "^1.0.2", "@actions/tool-cache": "^1.5.5", + "lodash.memoize": "^4.1.2", "semver": "^6.3.1" }, "devDependencies": { "@types/jest": "^27.0.2", + "@types/lodash": "^4.14.198", "@types/node": "^16.11.25", "@types/semver": "^6.0.0", "@typescript-eslint/eslint-plugin": "^5.54.0", diff --git a/src/cache-restore.ts b/src/cache-restore.ts index 183df9ea5..2d65a2255 100644 --- a/src/cache-restore.ts +++ b/src/cache-restore.ts @@ -6,9 +6,10 @@ import fs from 'fs'; import {State, Outputs} from './constants'; import {PackageManagerInfo} from './package-managers'; -import {getCacheDirectoryPath, getPackageManagerInfo} from './cache-utils'; +import {getBuildCachePath, getCacheDirectoryPath, getPackageManagerInfo} from './cache-utils'; +import {getInput} from "@actions/core"; -export const restoreCache = async ( +export const restoreModCache = async ( versionSpec: string, packageManager: string, cacheDependencyPath?: string @@ -25,27 +26,68 @@ export const restoreCache = async ( if (!fileHash) { throw new Error( - 'Some specified paths were not resolved, unable to cache dependencies.' + 'Some specified paths were not resolved, unable to cache modules.' ); } const linuxVersion = process.env.RUNNER_OS === 'Linux' ? `${process.env.ImageOS}-` : ''; - const primaryKey = `setup-go-${platform}-${linuxVersion}go-${versionSpec}-${fileHash}`; - core.debug(`primary key is ${primaryKey}`); + const cacheIdInput = getInput('cache-id') + const cacheId = cacheIdInput ? `${cacheIdInput}-` : '' + const primaryKey = `setup-go-${platform}-${linuxVersion}go-${versionSpec}-${cacheId}${fileHash}`; + core.debug(`Primary key for modules cache is ${primaryKey}`); - core.saveState(State.CachePrimaryKey, primaryKey); + core.saveState(State.CacheModPrimaryKey, primaryKey); const cacheKey = await cache.restoreCache(cachePaths, primaryKey); - core.setOutput(Outputs.CacheHit, Boolean(cacheKey)); + core.setOutput(Outputs.CacheModHit, Boolean(cacheKey)); + + if (!cacheKey) { + core.info(`Modules cache is not found`); + core.setOutput(Outputs.CacheModHit, false); + return; + } + + core.saveState(State.CacheModMatchedKey, cacheKey); + core.info(`Modules cache restored from key: ${cacheKey}`); +}; + +export const restoreBuildCache = async ( + versionSpec: string, + cacheBuildPath: string +) => { + const platform = process.env.RUNNER_OS; + + const cachePath = await getBuildCachePath() + + const fileHash = await glob.hashFiles(cacheBuildPath); + + if (!fileHash) { + throw new Error( + `The paths ${cacheBuildPath} were not resolved, unable to cache intermediate build files.` + ); + } + + const linuxVersion = + process.env.RUNNER_OS === 'Linux' ? `${process.env.ImageOS}-` : ''; + const cacheIdInput = getInput('cache-id') + const cacheId = cacheIdInput ? `${cacheIdInput}-` : '' + const keyPrefix = `setup-go-build-${platform}-${linuxVersion}go-${versionSpec}-${cacheId}`; + const primaryKey = `${keyPrefix}-${fileHash}`; + core.debug(`Primary key for intermediate build files cache is ${primaryKey}`); + + core.saveState(State.CacheBuildPrimaryKey, primaryKey); + + const cacheKey = await cache.restoreCache([cachePath], primaryKey, [keyPrefix]); + core.setOutput(Outputs.CacheBuildHit, Boolean(cacheKey)); if (!cacheKey) { core.info(`Cache is not found`); - core.setOutput(Outputs.CacheHit, false); + core.setOutput(Outputs.CacheBuildHit, false); return; } - core.saveState(State.CacheMatchedKey, cacheKey); + core.saveState(State.CacheBuildMatchedKey, cacheKey); core.info(`Cache restored from key: ${cacheKey}`); }; diff --git a/src/cache-save.ts b/src/cache-save.ts index 584d0a697..021af15d4 100644 --- a/src/cache-save.ts +++ b/src/cache-save.ts @@ -2,7 +2,13 @@ import * as core from '@actions/core'; import * as cache from '@actions/cache'; import fs from 'fs'; import {State} from './constants'; -import {getCacheDirectoryPath, getPackageManagerInfo} from './cache-utils'; +import { + getBuildCachePath, + getCacheDirectoryPath, + getPackageManagerInfo, + needBuildCache, + needModCache +} from './cache-utils'; // Catch and log any unhandled exceptions. These exceptions can leak out of the uploadChunk method in // @actions/toolkit when a failed upload closes the file descriptor causing any in-process reads to @@ -13,8 +19,11 @@ process.on('uncaughtException', e => { }); export async function run() { + if (core.getInput('cache-lookup-only').toLowerCase() === 'true') + return; try { await cachePackages(); + await cacheBuild(); } catch (error) { let message = 'Unknown error!'; if (error instanceof Error) { @@ -28,15 +37,15 @@ export async function run() { } const cachePackages = async () => { - const cacheInput = core.getBooleanInput('cache'); - if (!cacheInput) { + const needCache = needModCache() + if (!needCache) { return; } const packageManager = 'default'; - const state = core.getState(State.CacheMatchedKey); - const primaryKey = core.getState(State.CachePrimaryKey); + const state = core.getState(State.CacheModMatchedKey); + const primaryKey = core.getState(State.CacheModPrimaryKey); const packageManagerInfo = await getPackageManagerInfo(packageManager); @@ -80,6 +89,48 @@ const cachePackages = async () => { core.info(`Cache saved with the key: ${primaryKey}`); }; +const cacheBuild = async () => { + const needCache = needBuildCache() + if (!needCache) { + return; + } + + const state = core.getState(State.CacheBuildMatchedKey); + const primaryKey = core.getState(State.CacheBuildPrimaryKey); + + const cachePath = await getBuildCachePath() + + if (!fs.existsSync(cachePath)) { + core.warning('There are no intermediate build files cache folders on the disk'); + return; + } + + if (!fs.existsSync(cachePath)) { + logWarning( `Cache folder path is retrieved but doesn't exist on disk: ${cachePath}` ); + return; + } + + if (!primaryKey) { + core.info( + 'Primary key for intermediate build files cache was not generated. Please check the log messages above for more errors or information' + ); + return; + } + + if (primaryKey === state) { + core.info( + `Cache hit occurred on the primary key ${primaryKey} for intermediate build files cache, not saving cache.` + ); + return; + } + + const cacheId = await cache.saveCache([cachePath], primaryKey); + if (cacheId === -1) { + return; + } + core.info(`Cache saved with the key: ${primaryKey}`); +}; + function logWarning(message: string): void { const warningPrefix = '[warning]'; core.info(`${warningPrefix}${message}`); diff --git a/src/cache-utils.ts b/src/cache-utils.ts index 545c97af1..f3dad1b1d 100644 --- a/src/cache-utils.ts +++ b/src/cache-utils.ts @@ -1,85 +1,124 @@ import * as cache from '@actions/cache'; import * as core from '@actions/core'; import * as exec from '@actions/exec'; +import * as _ from 'lodash' import {supportedPackageManagers, PackageManagerInfo} from './package-managers'; +import * as glob from "@actions/glob"; export const getCommandOutput = async (toolCommand: string) => { - let {stdout, stderr, exitCode} = await exec.getExecOutput( - toolCommand, - undefined, - {ignoreReturnCode: true} - ); - - if (exitCode) { - stderr = !stderr.trim() - ? `The '${toolCommand}' command failed with exit code: ${exitCode}` - : stderr; - throw new Error(stderr); - } - - return stdout.trim(); + let {stdout, stderr, exitCode} = await exec.getExecOutput( + toolCommand, + undefined, + {ignoreReturnCode: true} + ); + + if (exitCode) { + stderr = !stderr.trim() + ? `The '${toolCommand}' command failed with exit code: ${exitCode}` + : stderr; + throw new Error(stderr); + } + + return stdout.trim(); }; export const getPackageManagerInfo = async (packageManager: string) => { - if (!supportedPackageManagers[packageManager]) { - throw new Error( - `It's not possible to use ${packageManager}, please, check correctness of the package manager name spelling.` - ); - } - const obtainedPackageManager = supportedPackageManagers[packageManager]; + if (!supportedPackageManagers[packageManager]) { + throw new Error( + `It's not possible to use ${packageManager}, please, check correctness of the package manager name spelling.` + ); + } + const obtainedPackageManager = supportedPackageManagers[packageManager]; - return obtainedPackageManager; + return obtainedPackageManager; }; export const getCacheDirectoryPath = async ( - packageManagerInfo: PackageManagerInfo + packageManagerInfo: PackageManagerInfo ) => { - const pathOutputs = await Promise.allSettled( - packageManagerInfo.cacheFolderCommandList.map(async command => - getCommandOutput(command) - ) - ); - - const results = pathOutputs.map(item => { - if (item.status === 'fulfilled') { - return item.value; - } else { - core.info(`[warning]getting cache directory path failed: ${item.reason}`); - } + const pathOutputs = await Promise.allSettled( + packageManagerInfo.cacheFolderCommandList.map(async command => + getCommandOutput(command) + ) + ); + + const results = pathOutputs.map(item => { + if (item.status === 'fulfilled') { + return item.value; + } else { + core.info(`[warning]getting cache directory path failed: ${item.reason}`); + } - return ''; - }); + return ''; + }); - const cachePaths = results.filter(item => item); + const cachePaths = results.filter(item => item); - if (!cachePaths.length) { - throw new Error(`Could not get cache folder paths.`); - } + if (!cachePaths.length) { + throw new Error(`Could not get cache folder paths.`); + } - return cachePaths; + return cachePaths; }; export function isGhes(): boolean { - const ghUrl = new URL( - process.env['GITHUB_SERVER_URL'] || 'https://github.com' - ); - return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM'; + const ghUrl = new URL( + process.env['GITHUB_SERVER_URL'] || 'https://github.com' + ); + return ghUrl.hostname.toUpperCase() !== 'GITHUB.COM'; } +/** + * Memoize it in order to avoid confusing multiple messages + */ export function isCacheFeatureAvailable(): boolean { - if (cache.isFeatureAvailable()) { - return true; - } + if (cache.isFeatureAvailable()) { + return true; + } + + if (isGhes()) { + core.warning( + 'Cache action is only supported on GHES version >= 3.5. If you are on version >=3.5 Please check with GHES admin if Actions cache service is enabled or not.' + ); + return false; + } - if (isGhes()) { core.warning( - 'Cache action is only supported on GHES version >= 3.5. If you are on version >=3.5 Please check with GHES admin if Actions cache service is enabled or not.' + 'The runner was not able to contact the cache service. Caching will be skipped' ); return false; - } +} + +/** + * Checks if the caching of dependencies is requested + * - `cache-mod` input takes precedence over `cache` input if set + */ +export function needModCache(): Boolean { + const cache = core.getBooleanInput('cache'); + const modCache = core.getInput('cache-mod').toLowerCase() + + return (modCache === 'true' || cache && modCache !== 'false') +} - core.warning( - 'The runner was not able to contact the cache service. Caching will be skipped' - ); - return false; +/** + * Checks if the caching of intermediate build files is requested + * - `cache-mod` input takes precedence over `cache` input if set + */ +export function needBuildCache(): Boolean { + const cache = core.getBooleanInput('cache'); + const buildCache = core.getInput('cache-build').toLowerCase() + + return (buildCache === 'true' || cache && buildCache !== 'false') +} + +export function getModDependenciesPath(): string { + return core.getInput('cache-dependency-path') } + +export function getBuildDependenciesPath(): string { + return core.getInput('cache-build-path') || "**/*.go" +} + +export function getBuildCachePath(): Promise { + return getCommandOutput('go env GOCACHE') +} \ No newline at end of file diff --git a/src/constants.ts b/src/constants.ts index b43d18c00..0e606d706 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,8 +1,11 @@ export enum State { - CachePrimaryKey = 'CACHE_KEY', - CacheMatchedKey = 'CACHE_RESULT' + CacheModPrimaryKey = 'CACHE_KEY', + CacheModMatchedKey = 'CACHE_RESULT', + CacheBuildPrimaryKey = 'CACHE_BUILD_KEY', + CacheBuildMatchedKey = 'CACHE_BUILD_RESULT' } export enum Outputs { - CacheHit = 'cache-hit' + CacheModHit = 'cache-hit', + CacheBuildHit = 'cache-build-hit' } diff --git a/src/main.ts b/src/main.ts index d3fb857df..3bc501d0c 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,8 +3,14 @@ import * as io from '@actions/io'; import * as installer from './installer'; import * as semver from 'semver'; import path from 'path'; -import {restoreCache} from './cache-restore'; -import {isCacheFeatureAvailable} from './cache-utils'; +import {restoreBuildCache, restoreModCache} from './cache-restore'; +import { + getBuildDependenciesPath, + getModDependenciesPath, + isCacheFeatureAvailable, + needBuildCache, + needModCache +} from './cache-utils'; import cp from 'child_process'; import fs from 'fs'; import os from 'os'; @@ -17,7 +23,6 @@ export async function run() { // const versionSpec = resolveVersionInput(); - const cache = core.getBooleanInput('cache'); core.info(`Setup go version spec ${versionSpec}`); let arch = core.getInput('architecture'); @@ -64,17 +69,30 @@ export async function run() { const goPath = await io.which('go'); const goVersion = (cp.execSync(`${goPath} version`) || '').toString(); - if (cache && isCacheFeatureAvailable()) { + const cacheFeatureAvailable = isCacheFeatureAvailable() + if (needModCache() && cacheFeatureAvailable) { const packageManager = 'default'; - const cacheDependencyPath = core.getInput('cache-dependency-path'); + const cacheDependencyPath = getModDependenciesPath() try { - await restoreCache( + await restoreModCache( parseGoVersion(goVersion), packageManager, cacheDependencyPath ); } catch (error) { - core.warning(`Restore cache failed: ${error.message}`); + core.warning(`Restore modules cache failed: ${error.message}`); + } + } + + if (needBuildCache() && cacheFeatureAvailable) { + const cacheBuildPath = getBuildDependenciesPath() + try { + await restoreBuildCache( + parseGoVersion(goVersion), + cacheBuildPath + ); + } catch (error) { + core.warning(`Restore modules cache failed: ${error.message}`); } } @@ -110,7 +128,7 @@ export async function addBinToPath(): Promise { const gp = buf.toString().trim(); core.debug(`go env GOPATH :${gp}:`); if (!fs.existsSync(gp)) { - // some of the hosted images have go install but not profile dir + // some of hosted images have go install but not profile dir core.debug(`creating ${gp}`); await io.mkdirP(gp); } diff --git a/src/package-managers.ts b/src/package-managers.ts index 3547d3366..bbbf5b666 100644 --- a/src/package-managers.ts +++ b/src/package-managers.ts @@ -10,6 +10,6 @@ export interface PackageManagerInfo { export const supportedPackageManagers: SupportedPackageManagers = { default: { dependencyFilePattern: 'go.sum', - cacheFolderCommandList: ['go env GOMODCACHE', 'go env GOCACHE'] + cacheFolderCommandList: ['go env GOMODCACHE'] } };