Skip to content

Commit

Permalink
[DEVX-1330] Limit test execution time to 30 mins (#161)
Browse files Browse the repository at this point in the history 
* add timeout

* update snapshot

* remove unused comments

* review comments

* update snapshot

* fix timeout
  • Loading branch information
@valentyn88
valentyn88 authored Nov 8, 2021
1 parent 351abaf commit 5a3ec67
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 16 deletions.
4 changes: 3 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,4 +126,6 @@ tests/*/cypress/videos/
node

Cache/
.DS_Store
.DS_Store

.idea/
34 changes: 25 additions & 9 deletions src/cypress-runner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ const cypress = require('cypress');
const util = require('util');
const _ = require('lodash');

const report = async (results, browserName, runCfg, suiteName, startTime, endTime, metrics) => {
const report = async (results = {}, statusCode, browserName, runCfg, suiteName, startTime, endTime, metrics) => {
// Prepare the assets
const runs = results.runs || [];
let specFiles = runs.map((run) => run.spec.name);
Expand All @@ -28,20 +28,20 @@ const report = async (results, browserName, runCfg, suiteName, startTime, endTim
browserName,
platformName,
);
const passed = failures === 0 && statusCode === 0;
// Run in cloud mode
if (process.env.SAUCE_VM) {
return failures === 0;
return passed;
}
if (!(process.env.SAUCE_USERNAME && process.env.SAUCE_ACCESS_KEY)) {
console.log('Skipping asset uploads! Remember to setup your SAUCE_USERNAME/SAUCE_ACCESS_KEY');
return failures === 0;
return passed;
}
// Run in docker mode
if (process.env.SAUCE_USERNAME !== '' && process.env.SAUCE_ACCESS_KEY !== '') {
await sauceReporter(runCfg, suiteName, browserName, assets, failures, startTime, endTime);
}

return failures === 0;
return passed;
};

// Configure reporters
Expand Down Expand Up @@ -140,7 +140,7 @@ const canAccessFolder = async function (file) {
await fsAccess(file, fs.constants.R_OK | fs.constants.W_OK);
};

const cypressRunner = async function (runCfgPath, suiteName) {
const cypressRunner = async function (runCfgPath, suiteName, timeoutSec) {
runCfgPath = getAbsolutePath(runCfgPath);
const runCfg = await loadRunConfig(runCfgPath);
runCfg.path = runCfgPath;
Expand All @@ -158,18 +158,34 @@ const cypressRunner = async function (runCfgPath, suiteName) {
metrics.push(npmMetrics);
let cypressOpts = getCypressOpts(runCfg, suiteName);
let startTime = new Date().toISOString();
const results = await cypress.run(cypressOpts);
const suites = runCfg.suites || [];
const suite = suites.find((testSuite) => testSuite.name === suiteName);
// saucectl suite.timeout is in nanoseconds
timeoutSec = suite.timeout / 1000000000 || timeoutSec;
let timeout;
const timeoutPromise = new Promise((resolve) => {
timeout = setTimeout(() => {
console.error(`Test timed out after ${timeoutSec} seconds`);
resolve();
}, timeoutSec * 1000);
});

let results = await Promise.race([timeoutPromise, cypress.run(cypressOpts)]);
clearTimeout(timeout);
const statusCode = results ? 0 : 1;
let endTime = new Date().toISOString();

return await report(results, cypressOpts.browser, runCfg, suiteName, startTime, endTime, metrics);
return await report(results, statusCode, cypressOpts.browser, runCfg, suiteName, startTime, endTime, metrics);
};

// For dev and test purposes, this allows us to run our Cypress Runner from command line
if (require.main === module) {
console.log(`Sauce Cypress Runner ${require(path.join(__dirname, '..', 'package.json')).version}`);
const { runCfgPath, suiteName } = getArgs();
// maxTimeout maximum test execution timeout is 1800 seconds (30 mins)
const maxTimeout = 1800;

cypressRunner(runCfgPath, suiteName)
cypressRunner(runCfgPath, suiteName, maxTimeout)
// eslint-disable-next-line promise/prefer-await-to-then
.then((passed) => process.exit(passed ? 0 : 1))
// eslint-disable-next-line promise/prefer-await-to-callbacks
Expand Down
30 changes: 24 additions & 6 deletions tests/unit/src/cypress-runner.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,20 @@ describe('.cypressRunner', function () {
afterEach(function () {
SauceReporter.sauceReporter.mockReset();
});
it('returns failure if Cypress.run is called with a timeout of 0 (Docker mode)', async function () {
const run = new Promise((resolve) => {
setTimeout(resolve, 10);
});
cypress.run.mockImplementation(() => run);
process.env.SAUCE_USERNAME = 'fake-sauce-username';
process.env.SAUCE_ACCESS_KEY = 'fake-sauce-accesskey';
const status = await cypressRunner('/fake/runner/path', 'fake-suite', 2);
expect(status).toEqual(false);
});
it('can call Cypress.run with basic args', async function () {
process.env.SAUCE_USERNAME = 'fake-sauce-username';
process.env.SAUCE_ACCESS_KEY = 'fake-sauce-accesskey';
await cypressRunner('/fake/runner/path', 'fake-suite');
await cypressRunner('/fake/runner/path', 'fake-suite', 1);
// Change reporter to not be fully-qualified path
cypressRunSpy.mock.calls[0][0].config.reporter = path.basename(cypressRunSpy.mock.calls[0][0].config.reporter);
cypressRunSpy.mock.calls[0][0].config.reporterOptions.configFile = path.basename(cypressRunSpy.mock.calls[0][0].config.reporterOptions.configFile);
Expand All @@ -61,15 +71,15 @@ describe('.cypressRunner', function () {
});
it('can hardcode the browser path', async function () {
process.env.SAUCE_BROWSER = 'C:/User/App/browser.exe:chrome';
await cypressRunner('/fake/runner/path', 'fake-suite');
await cypressRunner('/fake/runner/path', 'fake-suite', 1);
const calledBrowser = cypressRunSpy.mock.calls[0][0].browser;
expect(calledBrowser).toEqual('C:/User/App/browser.exe:chrome');
});
it('calls sauce reporter and returns a job status', async function () {
process.env.SAUCE_USERNAME = 'bruno.alassia';
process.env.SAUCE_ACCESS_KEY = 'i_l0ve_mayonnaise';
process.env.SAUCE_BROWSER = 'firefox';
await cypressRunner('/fake/runner/path', 'fake-suite');
await cypressRunner('/fake/runner/path', 'fake-suite', 1);
expect(SauceReporter.sauceReporter.mock.calls).toMatchSnapshot();
});
describe('from SAUCE VM', function () {
Expand All @@ -78,23 +88,31 @@ describe('.cypressRunner', function () {
});
it('returns false if there are test failures', async function () {
cypressRunSpy.mockImplementation(() => ({failures: 100}));
const status = await cypressRunner('/fake/runner/path', 'fake-suite');
const status = await cypressRunner('/fake/runner/path', 'fake-suite', 1);
expect(status).toEqual(false);
});
it('returns true if there are no test failures', async function () {
cypressRunSpy.mockImplementation(() => ({failures: 0}));
const status = await cypressRunner('/fake/runner/path', 'fake-suite');
const status = await cypressRunner('/fake/runner/path', 'fake-suite', 1);
expect(status).toEqual(false);
});
it('should take config.env as argument (DEVX-477)', async function () {
cypressRunSpy.mockImplementation(() => ({}));
await cypressRunner('/fake/runner/path', 'fake-suite');
await cypressRunner('/fake/runner/path', 'fake-suite', 1);
const { calls } = cypressRunSpy.mock;

// Rename to basename to remove home dir
calls[0][0].config.reporter = path.basename(calls[0][0].config.reporter);
calls[0][0].config.reporterOptions.configFile = path.basename(calls[0][0].config.reporterOptions.configFile);
expect(cypressRunSpy.mock.calls).toMatchSnapshot();
});
it('Cypress.run returns false if it times out (Sauce VM mode)', async function () {
const run = new Promise((resolve) => {
setTimeout(resolve, 10);
});
cypress.run.mockImplementation(() => run);
const status = await cypressRunner('/fake/runner/path', 'fake-suite', 1);
expect(status).toEqual(false);
});
});
});

0 comments on commit 5a3ec67

Please sign in to comment.