api/test-ci: add functionality for running all tests

api/src/util/test-ci.js

@@ -13,7 +13,56 @@ const getTests = (service) => loadJSON(getTestPath(service));
 
 // services that are known to frequently fail due to external
 // factors (e.g. rate limiting)
-const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube'])
+const finnicky = new Set(['bilibili', 'instagram', 'facebook', 'youtube']);
+
+const runTestsFor = async (service) => {
+    const tests = getTests(service);
+    let softFails = 0, fails = 0;
+
+    if (!tests) {
+        throw "no such service: " + service;
+    }
+
+    for (const test of tests) {
+        const { name, url, params, expected } = test;
+        const canFail = test.canFail || finnicky.has(service);
+
+        try {
+            await runTest(url, params, expected);
+            console.log(`${service}/${name}: ok`);
+
+        } catch(e) {
+            softFails += !canFail;
+            fails++;
+
+            let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL'));
+            if (canFail && process.env.GITHUB_ACTION) {
+                console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`);
+            }
+
+            console.error(`${service}/${name}: ${failText}`);
+            const errorString = e.toString().split('\n');
+            let c = '┃';
+            errorString.forEach((line, index) => {
+                line = line.replace('!=', Red('!='));
+
+                if (index === errorString.length - 1) {
+                    c = '┗';
+                }
+
+                console.error(`   ${c}`, line);
+            });
+        }
+    }
+
+    return { fails, softFails };
+}
+
+const printHeader = (service, padLen) => {
+    const padding = padLen - service.length;
+    service = service.padEnd(1 + service.length + padding, ' ');
+    console.log(service + '='.repeat(50));
+}
 
 const action = process.argv[2];
 switch (action) {
@@ -38,53 +87,45 @@ switch (action) {
 
     case "run-tests-for":
         const service = process.argv[3];
-        let failed = false;
-        const tests = getTests(service);
-
-        if (!tests) {
-            console.error('no such service:', service);
-            process.exitCode = 1;
-            break;
-        }
 
         env.streamLifespan = 10000;
-        env.apiURL = 'http://x';
+        env.apiURL = 'http://x/';
         randomizeCiphers();
 
-        for (const test of tests) {
-            const { name, url, params, expected } = test;
-            const canFail = test.canFail || finnicky.has(service);
-
         try {
-                await runTest(url, params, expected);
+            const { softFails } = await runTestsFor(service);
-                console.log(`${service}/${name}: ok`);
+            process.exitCode = Number(!!softFails);
-
         } catch(e) {
-                failed = !canFail;
+            console.error(e);
-
+            process.exitCode = 1;
-                let failText = canFail ? `${Red('FAIL')} (ignored)` : Bright(Red('FAIL'));
+            break;
-                if (canFail && process.env.GITHUB_ACTION) {
-                    console.log(`::warning title=${service}/${name.replace(/,/g, ';')}::failed and was ignored`);
         }
 
-                console.error(`${service}/${name}: ${failText}`);
-                const errorString = e.toString().split('\n');
-                let c = '┃';
-                errorString.forEach((line, index) => {
-                    line = line.replace('!=', Red('!='));
-
-                    if (index === errorString.length - 1) {
-                        c = '┗';
-                    }
-
-                    console.error(`   ${c}`, line);
-                });
-            }
-        }
-
-        process.exitCode = Number(failed);
         break;
     default:
-        console.error('invalid action:', action);
+        const maxHeaderLen = Object.keys(services).reduce((n, v) => v.length > n ? v.length : n, 0);
+        const failCounters = {};
+
+        env.streamLifespan = 10000;
+        env.apiURL = 'http://x/';
+        randomizeCiphers();
+
+        for (const service in services) {
+            printHeader(service, maxHeaderLen);
+            const { fails, softFails } = await runTestsFor(service);
+            failCounters[service] = fails;
+            console.log();
+
+            if (!process.exitCode && softFails)
                 process.exitCode = 1;
+        }
+
+        console.log('='.repeat(50 + maxHeaderLen));
+        console.log(
+            Bright('total fails:'),
+            Object.values(failCounters).reduce((a, b) => a + b)
+        );
+        for (const [ service, fails ] of Object.entries(failCounters)) {
+            if (fails) console.log(`${Bright(service)} fails: ${fails}`);
+        }
 }