Skip to content

Commit

Permalink
Fix Install on Windows is very slow (#393)
Browse files Browse the repository at this point in the history
* Fix Install on Windows is very slow

* Add unit test

* Improve readability

* Add e2e test

* fix lint

* Fix unit tests

* Fix unit tests

* limit to github hosted runners

* test hosted version of go

* AzDev environment

* rename lnkSrc

* refactor conditions

* improve tests

* refactoring

* Fix e2e test

* improve isHosted readability
  • Loading branch information
dsame authored Aug 3, 2023
1 parent 27eec5b commit 93397be
Show file tree
Hide file tree
Showing 5 changed files with 291 additions and 12 deletions.
114 changes: 114 additions & 0 deletions .github/workflows/windows-validation.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
name: Validate Windows installation

on:
push:
branches:
- main
paths-ignore:
- '**.md'
pull_request:
paths-ignore:
- '**.md'

jobs:
create-link-if-not-default:
runs-on: windows-latest
name: 'Validate if symlink is created'
strategy:
matrix:
cache: [false, true]
go: [1.20.1]
steps:
- uses: actions/checkout@v3

- name: 'Setup ${{ matrix.cache }}, cache: ${{ matrix.go }}'
uses: ./
with:
go-version: ${{ matrix.go }}
cache: ${{ matrix.cache }}

- name: 'Drive C: should have zero size link'
run: |
du -m -s 'C:\hostedtoolcache\windows\go\${{ matrix.go }}\x64'
# make sure drive c: contains only a link
size=$(du -m -s 'C:\hostedtoolcache\windows\go\${{ matrix.go }}\x64'|cut -f1 -d$'\t')
if [ $size -ne 0 ];then
echo 'Size of the link created on drive c: must be 0'
exit 1
fi
shell: bash

# Drive D: is small, take care the action does not eat up the space
- name: 'Drive D: space usage should be below 1G'
run: |
du -m -s 'D:\hostedtoolcache\windows\go\${{ matrix.go }}\x64'
size=$(du -m -s 'D:\hostedtoolcache\windows\go\${{ matrix.go }}\x64'|cut -f1 -d$'\t')
# make sure archive does not take lot of space
if [ $size -gt 999 ];then
echo 'Size of installed on drive d: go is too big';
exit 1
fi
shell: bash

# make sure the Go installation has not been changed to the end user
- name: Test paths and environments
run: |
echo $PATH
which go
go version
go env
if [ $(which go) != '/c/hostedtoolcache/windows/go/${{ matrix.go }}/x64/bin/go' ];then
echo 'which go should return "/c/hostedtoolcache/windows/go/${{ matrix.go }}/x64/bin/go"'
exit 1
fi
if [ $(go env GOROOT) != 'C:\hostedtoolcache\windows\go\${{ matrix.go }}\x64' ];then
echo 'go env GOROOT should return "C:\hostedtoolcache\windows\go\${{ matrix.go }}\x64"'
exit 1
fi
shell: bash

find-default-go:
name: 'Find default go version'
runs-on: windows-latest
outputs:
version: ${{ steps.goversion.outputs.version }}
steps:
- run: |
version=`go env GOVERSION|sed s/^go//`
echo "default go version: $version"
echo "version=$version" >> "$GITHUB_OUTPUT"
id: goversion
shell: bash
dont-create-link-if-default:
name: 'Validate if symlink is not created for default go'
runs-on: windows-latest
needs: find-default-go
strategy:
matrix:
cache: [false, true]
steps:
- uses: actions/checkout@v3

- name: 'Setup default go, cache: ${{ matrix.cache }}'
uses: ./
with:
go-version: ${{ needs.find-default-go.outputs.version }}
cache: ${{ matrix.cache }}

- name: 'Drive C: should have Go installation, cache: ${{ matrix.cache}}'
run: |
size=$(du -m -s 'C:\hostedtoolcache\windows\go\${{ needs.find-default-go.outputs.version }}\x64'|cut -f1 -d$'\t')
if [ $size -eq 0 ];then
echo 'Size of the hosted go installed on drive c: must be above zero'
exit 1
fi
shell: bash

- name: 'Drive D: should not have Go installation, cache: ${{ matrix.cache}}'
run: |
if [ -e 'D:\hostedtoolcache\windows\go\${{ needs.find-default-go.outputs.version }}\x64' ];then
echo 'D:\hostedtoolcache\windows\go\${{ needs.find-default-go.outputs.version }}\x64 should not exist for hosted version of go';
exit 1
fi
shell: bash
11 changes: 10 additions & 1 deletion __tests__/setup-go.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import * as io from '@actions/io';
import * as tc from '@actions/tool-cache';
import fs from 'fs';
import cp from 'child_process';
import osm from 'os';
import osm, {type} from 'os';
import path from 'path';
import * as main from '../src/main';
import * as im from '../src/installer';
Expand All @@ -16,6 +16,8 @@ const matcherRegExp = new RegExp(matcherPattern.regexp);
const win32Join = path.win32.join;
const posixJoin = path.posix.join;

jest.setTimeout(10000);

describe('setup-go', () => {
let inputs = {} as any;
let os = {} as any;
Expand All @@ -39,6 +41,8 @@ describe('setup-go', () => {
let existsSpy: jest.SpyInstance;
let readFileSpy: jest.SpyInstance;
let mkdirpSpy: jest.SpyInstance;
let mkdirSpy: jest.SpyInstance;
let symlinkSpy: jest.SpyInstance;
let execSpy: jest.SpyInstance;
let getManifestSpy: jest.SpyInstance;
let getAllVersionsSpy: jest.SpyInstance;
Expand Down Expand Up @@ -92,6 +96,11 @@ describe('setup-go', () => {
readFileSpy = jest.spyOn(fs, 'readFileSync');
mkdirpSpy = jest.spyOn(io, 'mkdirP');

// fs
mkdirSpy = jest.spyOn(fs, 'mkdir');
symlinkSpy = jest.spyOn(fs, 'symlinkSync');
symlinkSpy.mockImplementation(() => {});

// gets
getManifestSpy.mockImplementation(() => <tc.IToolRelease[]>goTestManifest);

Expand Down
62 changes: 62 additions & 0 deletions __tests__/windows-toolcache.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
import fs from 'fs';
import * as io from '@actions/io';
import * as tc from '@actions/tool-cache';
import path from 'path';

describe('Windows performance workaround', () => {
let mkdirSpy: jest.SpyInstance;
let symlinkSpy: jest.SpyInstance;
let statSpy: jest.SpyInstance;
let readdirSpy: jest.SpyInstance;
let writeFileSpy: jest.SpyInstance;
let rmRFSpy: jest.SpyInstance;
let mkdirPSpy: jest.SpyInstance;
let cpSpy: jest.SpyInstance;

let runnerToolCache: string | undefined;
beforeEach(() => {
mkdirSpy = jest.spyOn(fs, 'mkdir');
symlinkSpy = jest.spyOn(fs, 'symlinkSync');
statSpy = jest.spyOn(fs, 'statSync');
readdirSpy = jest.spyOn(fs, 'readdirSync');
writeFileSpy = jest.spyOn(fs, 'writeFileSync');
rmRFSpy = jest.spyOn(io, 'rmRF');
mkdirPSpy = jest.spyOn(io, 'mkdirP');
cpSpy = jest.spyOn(io, 'cp');

// default implementations
// @ts-ignore - not implement unused methods
statSpy.mockImplementation(() => ({
isDirectory: () => true
}));
readdirSpy.mockImplementation(() => []);
writeFileSpy.mockImplementation(() => {});
mkdirSpy.mockImplementation(() => {});
symlinkSpy.mockImplementation(() => {});
rmRFSpy.mockImplementation(() => Promise.resolve());
mkdirPSpy.mockImplementation(() => Promise.resolve());
cpSpy.mockImplementation(() => Promise.resolve());

runnerToolCache = process.env['RUNNER_TOOL_CACHE'];
});
afterEach(() => {
jest.clearAllMocks();
process.env['RUNNER_TOOL_CACHE'] = runnerToolCache;
});
// cacheWindowsToolkitDir depends on implementation of tc.cacheDir
// with the assumption that target dir is passed by RUNNER_TOOL_CACHE environment variable
// Make sure the implementation has not been changed
it('addExecutablesToCache should depend on env[RUNNER_TOOL_CACHE]', async () => {
process.env['RUNNER_TOOL_CACHE'] = '/faked-hostedtoolcache1';
const cacheDir1 = await tc.cacheDir('/qzx', 'go', '1.2.3', 'arch');
expect(cacheDir1).toBe(
path.join('/', 'faked-hostedtoolcache1', 'go', '1.2.3', 'arch')
);

process.env['RUNNER_TOOL_CACHE'] = '/faked-hostedtoolcache2';
const cacheDir2 = await tc.cacheDir('/qzx', 'go', '1.2.3', 'arch');
expect(cacheDir2).toBe(
path.join('/', 'faked-hostedtoolcache2', 'go', '1.2.3', 'arch')
);
});
});
46 changes: 43 additions & 3 deletions dist/setup/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -61488,6 +61488,46 @@ function resolveVersionFromManifest(versionSpec, stable, auth, arch, manifest) {
}
});
}
// for github hosted windows runner handle latency of OS drive
// by avoiding write operations to C:
function cacheWindowsDir(extPath, tool, version, arch) {
return __awaiter(this, void 0, void 0, function* () {
if (os_1.default.platform() !== 'win32')
return false;
// make sure the action runs in the hosted environment
if (process.env['RUNNER_ENVIRONMENT'] !== 'github-hosted' &&
process.env['AGENT_ISSELFHOSTED'] === '1')
return false;
const defaultToolCacheRoot = process.env['RUNNER_TOOL_CACHE'];
if (!defaultToolCacheRoot)
return false;
if (!fs_1.default.existsSync('d:\\') || !fs_1.default.existsSync('c:\\'))
return false;
const actualToolCacheRoot = defaultToolCacheRoot
.replace('C:', 'D:')
.replace('c:', 'd:');
// make toolcache root to be on drive d:
process.env['RUNNER_TOOL_CACHE'] = actualToolCacheRoot;
const actualToolCacheDir = yield tc.cacheDir(extPath, tool, version, arch);
// create a link from c: to d:
const defaultToolCacheDir = actualToolCacheDir.replace(actualToolCacheRoot, defaultToolCacheRoot);
fs_1.default.mkdirSync(path.dirname(defaultToolCacheDir), { recursive: true });
fs_1.default.symlinkSync(actualToolCacheDir, defaultToolCacheDir, 'junction');
core.info(`Created link ${defaultToolCacheDir} => ${actualToolCacheDir}`);
// make outer code to continue using toolcache as if it were installed on c:
// restore toolcache root to default drive c:
process.env['RUNNER_TOOL_CACHE'] = defaultToolCacheRoot;
return defaultToolCacheDir;
});
}
function addExecutablesToToolCache(extPath, info, arch) {
return __awaiter(this, void 0, void 0, function* () {
const tool = 'go';
const version = makeSemver(info.resolvedVersion);
return ((yield cacheWindowsDir(extPath, tool, version, arch)) ||
(yield tc.cacheDir(extPath, tool, version, arch)));
});
}
function installGoVersion(info, auth, arch) {
return __awaiter(this, void 0, void 0, function* () {
core.info(`Acquiring ${info.resolvedVersion} from ${info.downloadUrl}`);
Expand All @@ -61503,9 +61543,9 @@ function installGoVersion(info, auth, arch) {
extPath = path.join(extPath, 'go');
}
core.info('Adding to the cache ...');
const cachedDir = yield tc.cacheDir(extPath, 'go', makeSemver(info.resolvedVersion), arch);
core.info(`Successfully cached go to ${cachedDir}`);
return cachedDir;
const toolCacheDir = yield addExecutablesToToolCache(extPath, info, arch);
core.info(`Successfully cached go to ${toolCacheDir}`);
return toolCacheDir;
});
}
function extractGoArchive(archivePath) {
Expand Down
70 changes: 62 additions & 8 deletions src/installer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,64 @@ async function resolveVersionFromManifest(
}
}

// for github hosted windows runner handle latency of OS drive
// by avoiding write operations to C:
async function cacheWindowsDir(
extPath: string,
tool: string,
version: string,
arch: string
): Promise<string | false> {
if (os.platform() !== 'win32') return false;

// make sure the action runs in the hosted environment
if (
process.env['RUNNER_ENVIRONMENT'] !== 'github-hosted' &&
process.env['AGENT_ISSELFHOSTED'] === '1'
)
return false;

const defaultToolCacheRoot = process.env['RUNNER_TOOL_CACHE'];
if (!defaultToolCacheRoot) return false;

if (!fs.existsSync('d:\\') || !fs.existsSync('c:\\')) return false;

const actualToolCacheRoot = defaultToolCacheRoot
.replace('C:', 'D:')
.replace('c:', 'd:');
// make toolcache root to be on drive d:
process.env['RUNNER_TOOL_CACHE'] = actualToolCacheRoot;

const actualToolCacheDir = await tc.cacheDir(extPath, tool, version, arch);

// create a link from c: to d:
const defaultToolCacheDir = actualToolCacheDir.replace(
actualToolCacheRoot,
defaultToolCacheRoot
);
fs.mkdirSync(path.dirname(defaultToolCacheDir), {recursive: true});
fs.symlinkSync(actualToolCacheDir, defaultToolCacheDir, 'junction');
core.info(`Created link ${defaultToolCacheDir} => ${actualToolCacheDir}`);

// make outer code to continue using toolcache as if it were installed on c:
// restore toolcache root to default drive c:
process.env['RUNNER_TOOL_CACHE'] = defaultToolCacheRoot;
return defaultToolCacheDir;
}

async function addExecutablesToToolCache(
extPath: string,
info: IGoVersionInfo,
arch: string
): Promise<string> {
const tool = 'go';
const version = makeSemver(info.resolvedVersion);
return (
(await cacheWindowsDir(extPath, tool, version, arch)) ||
(await tc.cacheDir(extPath, tool, version, arch))
);
}

async function installGoVersion(
info: IGoVersionInfo,
auth: string | undefined,
Expand All @@ -186,14 +244,10 @@ async function installGoVersion(
}

core.info('Adding to the cache ...');
const cachedDir = await tc.cacheDir(
extPath,
'go',
makeSemver(info.resolvedVersion),
arch
);
core.info(`Successfully cached go to ${cachedDir}`);
return cachedDir;
const toolCacheDir = await addExecutablesToToolCache(extPath, info, arch);
core.info(`Successfully cached go to ${toolCacheDir}`);

return toolCacheDir;
}

export async function extractGoArchive(archivePath: string): Promise<string> {
Expand Down

0 comments on commit 93397be

Please sign in to comment.