From 93397bea11091df50f3d7e59dc26a7711a8bcfbe Mon Sep 17 00:00:00 2001 From: Sergey Dolin Date: Thu, 3 Aug 2023 14:33:56 +0200 Subject: [PATCH] Fix Install on Windows is very slow (#393) * Fix Install on Windows is very slow * Add unit test * Improve readability * Add e2e test * fix lint * Fix unit tests * Fix unit tests * limit to github hosted runners * test hosted version of go * AzDev environment * rename lnkSrc * refactor conditions * improve tests * refactoring * Fix e2e test * improve isHosted readability --- .github/workflows/windows-validation.yml | 114 +++++++++++++++++++++++ __tests__/setup-go.test.ts | 11 ++- __tests__/windows-toolcache.test.ts | 62 ++++++++++++ dist/setup/index.js | 46 ++++++++- src/installer.ts | 70 ++++++++++++-- 5 files changed, 291 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/windows-validation.yml create mode 100644 __tests__/windows-toolcache.test.ts diff --git a/.github/workflows/windows-validation.yml b/.github/workflows/windows-validation.yml new file mode 100644 index 000000000..6890d07bc --- /dev/null +++ b/.github/workflows/windows-validation.yml @@ -0,0 +1,114 @@ +name: Validate Windows installation + +on: + push: + branches: + - main + paths-ignore: + - '**.md' + pull_request: + paths-ignore: + - '**.md' + +jobs: + create-link-if-not-default: + runs-on: windows-latest + name: 'Validate if symlink is created' + strategy: + matrix: + cache: [false, true] + go: [1.20.1] + steps: + - uses: actions/checkout@v3 + + - name: 'Setup ${{ matrix.cache }}, cache: ${{ matrix.go }}' + uses: ./ + with: + go-version: ${{ matrix.go }} + cache: ${{ matrix.cache }} + + - name: 'Drive C: should have zero size link' + run: | + du -m -s 'C:\hostedtoolcache\windows\go\${{ matrix.go }}\x64' + # make sure drive c: contains only a link + size=$(du -m -s 'C:\hostedtoolcache\windows\go\${{ matrix.go }}\x64'|cut -f1 -d$'\t') + if [ $size -ne 0 ];then + echo 'Size of the link created on drive c: must be 0' + exit 1 + fi + shell: bash + + # Drive D: is small, take care the action does not eat up the space + - name: 'Drive D: space usage should be below 1G' + run: | + du -m -s 'D:\hostedtoolcache\windows\go\${{ matrix.go }}\x64' + size=$(du -m -s 'D:\hostedtoolcache\windows\go\${{ matrix.go }}\x64'|cut -f1 -d$'\t') + # make sure archive does not take lot of space + if [ $size -gt 999 ];then + echo 'Size of installed on drive d: go is too big'; + exit 1 + fi + shell: bash + + # make sure the Go installation has not been changed to the end user + - name: Test paths and environments + run: | + echo $PATH + which go + go version + go env + if [ $(which go) != '/c/hostedtoolcache/windows/go/${{ matrix.go }}/x64/bin/go' ];then + echo 'which go should return "/c/hostedtoolcache/windows/go/${{ matrix.go }}/x64/bin/go"' + exit 1 + fi + if [ $(go env GOROOT) != 'C:\hostedtoolcache\windows\go\${{ matrix.go }}\x64' ];then + echo 'go env GOROOT should return "C:\hostedtoolcache\windows\go\${{ matrix.go }}\x64"' + exit 1 + fi + shell: bash + + find-default-go: + name: 'Find default go version' + runs-on: windows-latest + outputs: + version: ${{ steps.goversion.outputs.version }} + steps: + - run: | + version=`go env GOVERSION|sed s/^go//` + echo "default go version: $version" + echo "version=$version" >> "$GITHUB_OUTPUT" + id: goversion + shell: bash + + dont-create-link-if-default: + name: 'Validate if symlink is not created for default go' + runs-on: windows-latest + needs: find-default-go + strategy: + matrix: + cache: [false, true] + steps: + - uses: actions/checkout@v3 + + - name: 'Setup default go, cache: ${{ matrix.cache }}' + uses: ./ + with: + go-version: ${{ needs.find-default-go.outputs.version }} + cache: ${{ matrix.cache }} + + - name: 'Drive C: should have Go installation, cache: ${{ matrix.cache}}' + run: | + size=$(du -m -s 'C:\hostedtoolcache\windows\go\${{ needs.find-default-go.outputs.version }}\x64'|cut -f1 -d$'\t') + if [ $size -eq 0 ];then + echo 'Size of the hosted go installed on drive c: must be above zero' + exit 1 + fi + shell: bash + + - name: 'Drive D: should not have Go installation, cache: ${{ matrix.cache}}' + run: | + if [ -e 'D:\hostedtoolcache\windows\go\${{ needs.find-default-go.outputs.version }}\x64' ];then + echo 'D:\hostedtoolcache\windows\go\${{ needs.find-default-go.outputs.version }}\x64 should not exist for hosted version of go'; + exit 1 + fi + shell: bash diff --git a/__tests__/setup-go.test.ts b/__tests__/setup-go.test.ts index cea14f4e8..70f2166eb 100644 --- a/__tests__/setup-go.test.ts +++ b/__tests__/setup-go.test.ts @@ -3,7 +3,7 @@ import * as io from '@actions/io'; import * as tc from '@actions/tool-cache'; import fs from 'fs'; import cp from 'child_process'; -import osm from 'os'; +import osm, {type} from 'os'; import path from 'path'; import * as main from '../src/main'; import * as im from '../src/installer'; @@ -16,6 +16,8 @@ const matcherRegExp = new RegExp(matcherPattern.regexp); const win32Join = path.win32.join; const posixJoin = path.posix.join; +jest.setTimeout(10000); + describe('setup-go', () => { let inputs = {} as any; let os = {} as any; @@ -39,6 +41,8 @@ describe('setup-go', () => { let existsSpy: jest.SpyInstance; let readFileSpy: jest.SpyInstance; let mkdirpSpy: jest.SpyInstance; + let mkdirSpy: jest.SpyInstance; + let symlinkSpy: jest.SpyInstance; let execSpy: jest.SpyInstance; let getManifestSpy: jest.SpyInstance; let getAllVersionsSpy: jest.SpyInstance; @@ -92,6 +96,11 @@ describe('setup-go', () => { readFileSpy = jest.spyOn(fs, 'readFileSync'); mkdirpSpy = jest.spyOn(io, 'mkdirP'); + // fs + mkdirSpy = jest.spyOn(fs, 'mkdir'); + symlinkSpy = jest.spyOn(fs, 'symlinkSync'); + symlinkSpy.mockImplementation(() => {}); + // gets getManifestSpy.mockImplementation(() => goTestManifest); diff --git a/__tests__/windows-toolcache.test.ts b/__tests__/windows-toolcache.test.ts new file mode 100644 index 000000000..52fd692cb --- /dev/null +++ b/__tests__/windows-toolcache.test.ts @@ -0,0 +1,62 @@ +import fs from 'fs'; +import * as io from '@actions/io'; +import * as tc from '@actions/tool-cache'; +import path from 'path'; + +describe('Windows performance workaround', () => { + let mkdirSpy: jest.SpyInstance; + let symlinkSpy: jest.SpyInstance; + let statSpy: jest.SpyInstance; + let readdirSpy: jest.SpyInstance; + let writeFileSpy: jest.SpyInstance; + let rmRFSpy: jest.SpyInstance; + let mkdirPSpy: jest.SpyInstance; + let cpSpy: jest.SpyInstance; + + let runnerToolCache: string | undefined; + beforeEach(() => { + mkdirSpy = jest.spyOn(fs, 'mkdir'); + symlinkSpy = jest.spyOn(fs, 'symlinkSync'); + statSpy = jest.spyOn(fs, 'statSync'); + readdirSpy = jest.spyOn(fs, 'readdirSync'); + writeFileSpy = jest.spyOn(fs, 'writeFileSync'); + rmRFSpy = jest.spyOn(io, 'rmRF'); + mkdirPSpy = jest.spyOn(io, 'mkdirP'); + cpSpy = jest.spyOn(io, 'cp'); + + // default implementations + // @ts-ignore - not implement unused methods + statSpy.mockImplementation(() => ({ + isDirectory: () => true + })); + readdirSpy.mockImplementation(() => []); + writeFileSpy.mockImplementation(() => {}); + mkdirSpy.mockImplementation(() => {}); + symlinkSpy.mockImplementation(() => {}); + rmRFSpy.mockImplementation(() => Promise.resolve()); + mkdirPSpy.mockImplementation(() => Promise.resolve()); + cpSpy.mockImplementation(() => Promise.resolve()); + + runnerToolCache = process.env['RUNNER_TOOL_CACHE']; + }); + afterEach(() => { + jest.clearAllMocks(); + process.env['RUNNER_TOOL_CACHE'] = runnerToolCache; + }); + // cacheWindowsToolkitDir depends on implementation of tc.cacheDir + // with the assumption that target dir is passed by RUNNER_TOOL_CACHE environment variable + // Make sure the implementation has not been changed + it('addExecutablesToCache should depend on env[RUNNER_TOOL_CACHE]', async () => { + process.env['RUNNER_TOOL_CACHE'] = '/faked-hostedtoolcache1'; + const cacheDir1 = await tc.cacheDir('/qzx', 'go', '1.2.3', 'arch'); + expect(cacheDir1).toBe( + path.join('/', 'faked-hostedtoolcache1', 'go', '1.2.3', 'arch') + ); + + process.env['RUNNER_TOOL_CACHE'] = '/faked-hostedtoolcache2'; + const cacheDir2 = await tc.cacheDir('/qzx', 'go', '1.2.3', 'arch'); + expect(cacheDir2).toBe( + path.join('/', 'faked-hostedtoolcache2', 'go', '1.2.3', 'arch') + ); + }); +}); diff --git a/dist/setup/index.js b/dist/setup/index.js index af096a9a8..b0a3f4e6b 100644 --- a/dist/setup/index.js +++ b/dist/setup/index.js @@ -61488,6 +61488,46 @@ function resolveVersionFromManifest(versionSpec, stable, auth, arch, manifest) { } }); } +// for github hosted windows runner handle latency of OS drive +// by avoiding write operations to C: +function cacheWindowsDir(extPath, tool, version, arch) { + return __awaiter(this, void 0, void 0, function* () { + if (os_1.default.platform() !== 'win32') + return false; + // make sure the action runs in the hosted environment + if (process.env['RUNNER_ENVIRONMENT'] !== 'github-hosted' && + process.env['AGENT_ISSELFHOSTED'] === '1') + return false; + const defaultToolCacheRoot = process.env['RUNNER_TOOL_CACHE']; + if (!defaultToolCacheRoot) + return false; + if (!fs_1.default.existsSync('d:\\') || !fs_1.default.existsSync('c:\\')) + return false; + const actualToolCacheRoot = defaultToolCacheRoot + .replace('C:', 'D:') + .replace('c:', 'd:'); + // make toolcache root to be on drive d: + process.env['RUNNER_TOOL_CACHE'] = actualToolCacheRoot; + const actualToolCacheDir = yield tc.cacheDir(extPath, tool, version, arch); + // create a link from c: to d: + const defaultToolCacheDir = actualToolCacheDir.replace(actualToolCacheRoot, defaultToolCacheRoot); + fs_1.default.mkdirSync(path.dirname(defaultToolCacheDir), { recursive: true }); + fs_1.default.symlinkSync(actualToolCacheDir, defaultToolCacheDir, 'junction'); + core.info(`Created link ${defaultToolCacheDir} => ${actualToolCacheDir}`); + // make outer code to continue using toolcache as if it were installed on c: + // restore toolcache root to default drive c: + process.env['RUNNER_TOOL_CACHE'] = defaultToolCacheRoot; + return defaultToolCacheDir; + }); +} +function addExecutablesToToolCache(extPath, info, arch) { + return __awaiter(this, void 0, void 0, function* () { + const tool = 'go'; + const version = makeSemver(info.resolvedVersion); + return ((yield cacheWindowsDir(extPath, tool, version, arch)) || + (yield tc.cacheDir(extPath, tool, version, arch))); + }); +} function installGoVersion(info, auth, arch) { return __awaiter(this, void 0, void 0, function* () { core.info(`Acquiring ${info.resolvedVersion} from ${info.downloadUrl}`); @@ -61503,9 +61543,9 @@ function installGoVersion(info, auth, arch) { extPath = path.join(extPath, 'go'); } core.info('Adding to the cache ...'); - const cachedDir = yield tc.cacheDir(extPath, 'go', makeSemver(info.resolvedVersion), arch); - core.info(`Successfully cached go to ${cachedDir}`); - return cachedDir; + const toolCacheDir = yield addExecutablesToToolCache(extPath, info, arch); + core.info(`Successfully cached go to ${toolCacheDir}`); + return toolCacheDir; }); } function extractGoArchive(archivePath) { diff --git a/src/installer.ts b/src/installer.ts index 013fb6405..be90e101a 100644 --- a/src/installer.ts +++ b/src/installer.ts @@ -164,6 +164,64 @@ async function resolveVersionFromManifest( } } +// for github hosted windows runner handle latency of OS drive +// by avoiding write operations to C: +async function cacheWindowsDir( + extPath: string, + tool: string, + version: string, + arch: string +): Promise { + if (os.platform() !== 'win32') return false; + + // make sure the action runs in the hosted environment + if ( + process.env['RUNNER_ENVIRONMENT'] !== 'github-hosted' && + process.env['AGENT_ISSELFHOSTED'] === '1' + ) + return false; + + const defaultToolCacheRoot = process.env['RUNNER_TOOL_CACHE']; + if (!defaultToolCacheRoot) return false; + + if (!fs.existsSync('d:\\') || !fs.existsSync('c:\\')) return false; + + const actualToolCacheRoot = defaultToolCacheRoot + .replace('C:', 'D:') + .replace('c:', 'd:'); + // make toolcache root to be on drive d: + process.env['RUNNER_TOOL_CACHE'] = actualToolCacheRoot; + + const actualToolCacheDir = await tc.cacheDir(extPath, tool, version, arch); + + // create a link from c: to d: + const defaultToolCacheDir = actualToolCacheDir.replace( + actualToolCacheRoot, + defaultToolCacheRoot + ); + fs.mkdirSync(path.dirname(defaultToolCacheDir), {recursive: true}); + fs.symlinkSync(actualToolCacheDir, defaultToolCacheDir, 'junction'); + core.info(`Created link ${defaultToolCacheDir} => ${actualToolCacheDir}`); + + // make outer code to continue using toolcache as if it were installed on c: + // restore toolcache root to default drive c: + process.env['RUNNER_TOOL_CACHE'] = defaultToolCacheRoot; + return defaultToolCacheDir; +} + +async function addExecutablesToToolCache( + extPath: string, + info: IGoVersionInfo, + arch: string +): Promise { + const tool = 'go'; + const version = makeSemver(info.resolvedVersion); + return ( + (await cacheWindowsDir(extPath, tool, version, arch)) || + (await tc.cacheDir(extPath, tool, version, arch)) + ); +} + async function installGoVersion( info: IGoVersionInfo, auth: string | undefined, @@ -186,14 +244,10 @@ async function installGoVersion( } core.info('Adding to the cache ...'); - const cachedDir = await tc.cacheDir( - extPath, - 'go', - makeSemver(info.resolvedVersion), - arch - ); - core.info(`Successfully cached go to ${cachedDir}`); - return cachedDir; + const toolCacheDir = await addExecutablesToToolCache(extPath, info, arch); + core.info(`Successfully cached go to ${toolCacheDir}`); + + return toolCacheDir; } export async function extractGoArchive(archivePath: string): Promise {