diff --git a/api/src/misc/run-test.js b/api/src/misc/run-test.js index 10d19aef..21d97d04 100644 --- a/api/src/misc/run-test.js +++ b/api/src/misc/run-test.js @@ -41,4 +41,4 @@ export async function runTest(url, params, expect) { if (result.body.status === 'tunnel') { // TODO: stream testing } -} \ No newline at end of file +} diff --git a/api/src/util/test-ci.js b/api/src/util/test-ci.js index 5a1d6300..a8b70fd3 100644 --- a/api/src/util/test-ci.js +++ b/api/src/util/test-ci.js @@ -13,7 +13,56 @@ const getTests = (service) => loadJSON(getTestPath(service)); // services that are known to frequently fail due to external // factors (e.g. rate limiting) -const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube']) +const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube']); + +const runTestsFor = async (service) => { + const tests = getTests(service); + let softFails = 0, fails = 0; + + if (!tests) { + throw "no such service: " + service; + } + + for (const test of tests) { + const { name, url, params, expected } = test; + const canFail = test.canFail || finnicky.has(service); + + try { + await runTest(url, params, expected); + console.log(`${service}/${name}: ok`); + + } catch(e) { + softFails += !canFail; + fails++; + + let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL')); + if (canFail && process.env.GITHUB_ACTION) { + console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`); + } + + console.error(`${service}/${name}: ${failText}`); + const errorString = e.toString().split('\n'); + let c = '┃'; + errorString.forEach((line, index) => { + line = line.replace('!=', Red('!=')); + + if (index === errorString.length - 1) { + c = '┗'; + } + + console.error(` ${c}`, line); + }); + } + } + + return { fails, softFails }; +} + +const printHeader = (service, padLen) => { + const padding = padLen - service.length; + service = service.padEnd(1 + service.length + padding, ' '); + console.log(service + '='.repeat(50)); +} const action = process.argv[2]; switch (action) { @@ -38,53 +87,45 @@ switch (action) { case "run-tests-for": const service = process.argv[3]; - let failed = false; - const tests = getTests(service); - if (!tests) { - console.error('no such service:', service); + env.streamLifespan = 10000; + env.apiURL = 'http://x/'; + randomizeCiphers(); + + try { + const { softFails } = await runTestsFor(service); + process.exitCode = Number(!!softFails); + } catch(e) { + console.error(e); process.exitCode = 1; break; } - env.streamLifespan = 10000; - env.apiURL = 'http://x'; - randomizeCiphers(); - - for (const test of tests) { - const { name, url, params, expected } = test; - const canFail = test.canFail || finnicky.has(service); - - try { - await runTest(url, params, expected); - console.log(`${service}/${name}: ok`); - - } catch(e) { - failed = !canFail; - - let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL')); - if (canFail && process.env.GITHUB_ACTION) { - console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`); - } - - console.error(`${service}/${name}: ${failText}`); - const errorString = e.toString().split('\n'); - let c = '┃'; - errorString.forEach((line, index) => { - line = line.replace('!=', Red('!=')); - - if (index === errorString.length - 1) { - c = '┗'; - } - - console.error(` ${c}`, line); - }); - } - } - - process.exitCode = Number(failed); break; default: - console.error('invalid action:', action); - process.exitCode = 1; + const maxHeaderLen = Object.keys(services).reduce((n, v) => v.length > n ? v.length : n, 0); + const failCounters = {}; + + env.streamLifespan = 10000; + env.apiURL = 'http://x/'; + randomizeCiphers(); + + for (const service in services) { + printHeader(service, maxHeaderLen); + const { fails, softFails } = await runTestsFor(service); + failCounters[service] = fails; + console.log(); + + if (!process.exitCode && softFails) + process.exitCode = 1; + } + + console.log('='.repeat(50 + maxHeaderLen)); + console.log( + Bright('total fails:'), + Object.values(failCounters).reduce((a, b) => a + b) + ); + for (const [ service, fails ] of Object.entries(failCounters)) { + if (fails) console.log(`${Bright(service)} fails: ${fails}`); + } }