From 5a3ec67bb15e03a864ab998fb059cc2bd1db9e52 Mon Sep 17 00:00:00 2001 From: Valentyn Kremeshnyi Date: Mon, 8 Nov 2021 14:18:01 -0800 Subject: [PATCH] [DEVX-1330] Limit test execution time to 30 mins (#161) * add timeout * update snapshot * remove unused comments * review comments * update snapshot * fix timeout --- .gitignore | 4 +++- src/cypress-runner.js | 34 ++++++++++++++++++++------- tests/unit/src/cypress-runner.spec.js | 30 ++++++++++++++++++----- 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index e177b525..7b4cf142 100644 --- a/.gitignore +++ b/.gitignore @@ -126,4 +126,6 @@ tests/*/cypress/videos/ node Cache/ -.DS_Store \ No newline at end of file +.DS_Store + +.idea/ \ No newline at end of file diff --git a/src/cypress-runner.js b/src/cypress-runner.js index 76e50537..ab724973 100644 --- a/src/cypress-runner.js +++ b/src/cypress-runner.js @@ -6,7 +6,7 @@ const cypress = require('cypress'); const util = require('util'); const _ = require('lodash'); -const report = async (results, browserName, runCfg, suiteName, startTime, endTime, metrics) => { +const report = async (results = {}, statusCode, browserName, runCfg, suiteName, startTime, endTime, metrics) => { // Prepare the assets const runs = results.runs || []; let specFiles = runs.map((run) => run.spec.name); @@ -28,20 +28,20 @@ const report = async (results, browserName, runCfg, suiteName, startTime, endTim browserName, platformName, ); + const passed = failures === 0 && statusCode === 0; // Run in cloud mode if (process.env.SAUCE_VM) { - return failures === 0; + return passed; } if (!(process.env.SAUCE_USERNAME && process.env.SAUCE_ACCESS_KEY)) { console.log('Skipping asset uploads! Remember to setup your SAUCE_USERNAME/SAUCE_ACCESS_KEY'); - return failures === 0; + return passed; } // Run in docker mode if (process.env.SAUCE_USERNAME !== '' && process.env.SAUCE_ACCESS_KEY !== '') { await sauceReporter(runCfg, suiteName, browserName, assets, failures, startTime, endTime); } - - return failures === 0; + return passed; }; // Configure reporters @@ -140,7 +140,7 @@ const canAccessFolder = async function (file) { await fsAccess(file, fs.constants.R_OK | fs.constants.W_OK); }; -const cypressRunner = async function (runCfgPath, suiteName) { +const cypressRunner = async function (runCfgPath, suiteName, timeoutSec) { runCfgPath = getAbsolutePath(runCfgPath); const runCfg = await loadRunConfig(runCfgPath); runCfg.path = runCfgPath; @@ -158,18 +158,34 @@ const cypressRunner = async function (runCfgPath, suiteName) { metrics.push(npmMetrics); let cypressOpts = getCypressOpts(runCfg, suiteName); let startTime = new Date().toISOString(); - const results = await cypress.run(cypressOpts); + const suites = runCfg.suites || []; + const suite = suites.find((testSuite) => testSuite.name === suiteName); + // saucectl suite.timeout is in nanoseconds + timeoutSec = suite.timeout / 1000000000 || timeoutSec; + let timeout; + const timeoutPromise = new Promise((resolve) => { + timeout = setTimeout(() => { + console.error(`Test timed out after ${timeoutSec} seconds`); + resolve(); + }, timeoutSec * 1000); + }); + + let results = await Promise.race([timeoutPromise, cypress.run(cypressOpts)]); + clearTimeout(timeout); + const statusCode = results ? 0 : 1; let endTime = new Date().toISOString(); - return await report(results, cypressOpts.browser, runCfg, suiteName, startTime, endTime, metrics); + return await report(results, statusCode, cypressOpts.browser, runCfg, suiteName, startTime, endTime, metrics); }; // For dev and test purposes, this allows us to run our Cypress Runner from command line if (require.main === module) { console.log(`Sauce Cypress Runner ${require(path.join(__dirname, '..', 'package.json')).version}`); const { runCfgPath, suiteName } = getArgs(); + // maxTimeout maximum test execution timeout is 1800 seconds (30 mins) + const maxTimeout = 1800; - cypressRunner(runCfgPath, suiteName) + cypressRunner(runCfgPath, suiteName, maxTimeout) // eslint-disable-next-line promise/prefer-await-to-then .then((passed) => process.exit(passed ? 0 : 1)) // eslint-disable-next-line promise/prefer-await-to-callbacks diff --git a/tests/unit/src/cypress-runner.spec.js b/tests/unit/src/cypress-runner.spec.js index 2aacbbf9..fcb7f13d 100644 --- a/tests/unit/src/cypress-runner.spec.js +++ b/tests/unit/src/cypress-runner.spec.js @@ -49,10 +49,20 @@ describe('.cypressRunner', function () { afterEach(function () { SauceReporter.sauceReporter.mockReset(); }); + it('returns failure if Cypress.run is called with a timeout of 0 (Docker mode)', async function () { + const run = new Promise((resolve) => { + setTimeout(resolve, 10); + }); + cypress.run.mockImplementation(() => run); + process.env.SAUCE_USERNAME = 'fake-sauce-username'; + process.env.SAUCE_ACCESS_KEY = 'fake-sauce-accesskey'; + const status = await cypressRunner('/fake/runner/path', 'fake-suite', 2); + expect(status).toEqual(false); + }); it('can call Cypress.run with basic args', async function () { process.env.SAUCE_USERNAME = 'fake-sauce-username'; process.env.SAUCE_ACCESS_KEY = 'fake-sauce-accesskey'; - await cypressRunner('/fake/runner/path', 'fake-suite'); + await cypressRunner('/fake/runner/path', 'fake-suite', 1); // Change reporter to not be fully-qualified path cypressRunSpy.mock.calls[0][0].config.reporter = path.basename(cypressRunSpy.mock.calls[0][0].config.reporter); cypressRunSpy.mock.calls[0][0].config.reporterOptions.configFile = path.basename(cypressRunSpy.mock.calls[0][0].config.reporterOptions.configFile); @@ -61,7 +71,7 @@ describe('.cypressRunner', function () { }); it('can hardcode the browser path', async function () { process.env.SAUCE_BROWSER = 'C:/User/App/browser.exe:chrome'; - await cypressRunner('/fake/runner/path', 'fake-suite'); + await cypressRunner('/fake/runner/path', 'fake-suite', 1); const calledBrowser = cypressRunSpy.mock.calls[0][0].browser; expect(calledBrowser).toEqual('C:/User/App/browser.exe:chrome'); }); @@ -69,7 +79,7 @@ describe('.cypressRunner', function () { process.env.SAUCE_USERNAME = 'bruno.alassia'; process.env.SAUCE_ACCESS_KEY = 'i_l0ve_mayonnaise'; process.env.SAUCE_BROWSER = 'firefox'; - await cypressRunner('/fake/runner/path', 'fake-suite'); + await cypressRunner('/fake/runner/path', 'fake-suite', 1); expect(SauceReporter.sauceReporter.mock.calls).toMatchSnapshot(); }); describe('from SAUCE VM', function () { @@ -78,17 +88,17 @@ describe('.cypressRunner', function () { }); it('returns false if there are test failures', async function () { cypressRunSpy.mockImplementation(() => ({failures: 100})); - const status = await cypressRunner('/fake/runner/path', 'fake-suite'); + const status = await cypressRunner('/fake/runner/path', 'fake-suite', 1); expect(status).toEqual(false); }); it('returns true if there are no test failures', async function () { cypressRunSpy.mockImplementation(() => ({failures: 0})); - const status = await cypressRunner('/fake/runner/path', 'fake-suite'); + const status = await cypressRunner('/fake/runner/path', 'fake-suite', 1); expect(status).toEqual(false); }); it('should take config.env as argument (DEVX-477)', async function () { cypressRunSpy.mockImplementation(() => ({})); - await cypressRunner('/fake/runner/path', 'fake-suite'); + await cypressRunner('/fake/runner/path', 'fake-suite', 1); const { calls } = cypressRunSpy.mock; // Rename to basename to remove home dir @@ -96,5 +106,13 @@ describe('.cypressRunner', function () { calls[0][0].config.reporterOptions.configFile = path.basename(calls[0][0].config.reporterOptions.configFile); expect(cypressRunSpy.mock.calls).toMatchSnapshot(); }); + it('Cypress.run returns false if it times out (Sauce VM mode)', async function () { + const run = new Promise((resolve) => { + setTimeout(resolve, 10); + }); + cypress.run.mockImplementation(() => run); + const status = await cypressRunner('/fake/runner/path', 'fake-suite', 1); + expect(status).toEqual(false); + }); }); });