feat: better test reports

.github/scripts/create-e2e-test-report.ts

@@ -12,8 +12,14 @@ import {
   normalizeTestPath,
   XML,
 } from './shared/utils';
+import type { Endpoints } from '@octokit/types';
+
+type Job =
+  Endpoints['GET /repos/{owner}/{repo}/actions/runs/{run_id}/jobs']['response']['data']['jobs'][number];
 
 async function main() {
+  const { Octokit } = await import('octokit');
+
   const env = {
     OWNER: process.env.OWNER || 'metamask',
     REPOSITORY: process.env.REPOSITORY || 'metamask-extension',
@@ -23,11 +29,44 @@ async function main() {
     TEST_RESULTS_PATH: process.env.TEST_RESULTS_PATH || 'test/test-results/e2e',
     TEST_RUNS_PATH:
       process.env.TEST_RUNS_PATH || 'test/test-results/test-runs.json',
-    RUN_ID: process.env.RUN_ID ? +process.env.RUN_ID : 0,
-    PR_NUMBER: process.env.PR_NUMBER ? +process.env.PR_NUMBER : 0,
+    GITHUB_TOKEN: process.env.GITHUB_TOKEN!,
     GITHUB_ACTIONS: process.env.GITHUB_ACTIONS === 'true',
   };
 
+  const github = new Octokit({ auth: env.GITHUB_TOKEN });
+
+  const jobsCache: { [runId: number]: Job[] } = {};
+
+  async function getJobs(runId: number) {
+    if (!runId) {
+      return [];
+    } else if (jobsCache[runId]) {
+      return jobsCache[runId];
+    } else {
+      try {
+        const jobs = await github.paginate(
+          github.rest.actions.listJobsForWorkflowRun,
+          {
+            owner: env.OWNER,
+            repo: env.REPOSITORY,
+            run_id: runId,
+            per_page: 100,
+          },
+        );
+        jobsCache[runId] = jobs;
+        return jobsCache[runId];
+      } catch (error) {
+        return [];
+      }
+    }
+  }
+
+  async function getJobId(runId: number, jobName: string) {
+    const jobs = await getJobs(runId);
+    const job = jobs.find((job) => job.name.endsWith(jobName));
+    return job?.id;
+  }
+
   let summary = '';
   const core = env.GITHUB_ACTIONS
     ? await import('@actions/core')
@@ -41,6 +80,9 @@ async function main() {
         setFailed: (msg: string) => console.error(msg),
       };
 
+  const repositoryUrl = new URL('https://github.com');
+  repositoryUrl.pathname = `/${env.OWNER}/${env.REPOSITORY}`;
+
   try {
     const testRuns: TestRun[] = [];
     const filenames = await fs.readdir(env.TEST_RESULTS_PATH);
@@ -59,13 +101,20 @@ async function main() {
         const skipped = tests - suite.testcase.length;
         const passed = tests - failed - skipped;
 
+        const jobName = suite.properties?.[0].property?.[0]?.$.value
+          ? `${suite.properties?.[0].property?.[0]?.$.value}`
+          : '';
+        const runId = suite.properties?.[0].property?.[1]?.$.value
+          ? +suite.properties?.[0].property?.[1]?.$.value
+          : 0;
+        const jobId = (await getJobId(runId, jobName)) ?? 0;
+        const prNumber = suite.properties?.[0].property?.[2]?.$.value
+          ? +suite.properties?.[0].property?.[2]?.$.value
+          : 0;
+
         const testSuite: TestSuite = {
           name: suite.$.name,
-          job: {
-            name: suite.properties?.[0].property?.[0]?.$.value ?? '',
-            id: suite.properties?.[0].property?.[1]?.$.value ?? '',
-            runId: suite.properties?.[0].property?.[2]?.$.value ?? '',
-          },
+          job: { name: jobName, id: jobId, runId, prNumber },
           date: new Date(suite.$.timestamp),
           tests,
           passed,
@@ -221,18 +270,21 @@ async function main() {
           for (const file of testRun.testFiles) {
             if (file.failed === 0) continue;
             console.error(file.path);
-            core.summary.addRaw(
-              `\n#### [${file.path}](https://github.com/${env.OWNER}/${env.REPOSITORY}/blob/${env.BRANCH}/${file.path})\n`,
-            );
+            const testUrl = new URL(repositoryUrl);
+            testUrl.pathname += `/blob/${env.BRANCH}/${file.path}`;
+            core.summary.addRaw(`\n#### [${file.path}](${testUrl})\n`);
             for (const suite of file.testSuites) {
               if (suite.failed === 0) continue;
-              if (suite.job.name && suite.job.id && env.RUN_ID) {
+              if (suite.job.name && suite.job.id && suite.job.runId) {
+                const jobUrl = new URL(repositoryUrl);
+                jobUrl.pathname += `/actions/runs/${suite.job.runId}/job/${suite.job.id}`;
+                if (suite.job.prNumber) {
+                  jobUrl.search = new URLSearchParams({
+                    pr: suite.job.prNumber.toString(),
+                  }).toString();
+                }
                 core.summary.addRaw(
-                  `\n##### Job: [${suite.job.name}](https://github.com/${
-                    env.OWNER
-                  }/${env.REPOSITORY}/actions/runs/${env.RUN_ID}/job/${
-                    suite.job.id
-                  }${env.PR_NUMBER ? `?pr=${env.PR_NUMBER}` : ''})\n`,
+                  `\n##### Job: [${suite.job.name}](${jobUrl})\n`,
                 );
               }
               for (const test of suite.testCases) {
@@ -259,9 +311,11 @@ async function main() {
         const alignment = '| :--- | ---: | ---: | ---: | ---: |';
         const body = rows
           .map((row) => {
+            const testUrl = new URL(repositoryUrl);
+            testUrl.pathname += `/blob/${env.BRANCH}/${row['Test file']}`;
             const data = {
               ...row,
-              'Test file': `[${row['Test file']}](https://github.com/${env.OWNER}/${env.REPOSITORY}/blob/${env.BRANCH}/${row['Test file']})`,
+              'Test file': `[${row['Test file']}](${testUrl})`,
             };
             return `| ${Object.values(data).join(' | ')} |`;
           })

.github/scripts/create-flaky-test-report.ts

@@ -76,8 +76,10 @@ async function main() {
         });
         const buffer = Buffer.from(response.data as ArrayBuffer);
         const zip = await unzipper.Open.buffer(buffer);
-        const file = zip.files.find((file) => file.path === 'test-runs.json');
-        if (!file) throw new Error(`'test-runs.json' file in zip not found!`);
+        const file = zip.files.find((file) =>
+          file.path.startsWith('test-runs'),
+        );
+        if (!file) throw new Error(`test-runs file in zip not found!`);
         const content = await file.buffer();
         return JSON.parse(content.toString());
       }),
@@ -124,7 +126,9 @@ async function main() {
       }
       return summary;
     }, {}),
-  ).sort((a, b) => b.count - a.count);
+  )
+    .sort((a, b) => b.count - a.count)
+    .slice(0, 10);
 
   const options = { year: 'numeric', month: 'long', day: 'numeric' } as const;
   const fromDateString = from.toLocaleDateString('en-US', options);
@@ -137,7 +141,7 @@ async function main() {
   branchUrl.pathname += `/tree/${env.BRANCH}`;
 
   console.log(
-    `❌ Failed tests on the ${env.REPOSITORY} repository ${env.BRANCH} branch from ${fromDateString} to ${toDateString}`,
+    `❌ Top 10 failed tests on the ${env.REPOSITORY} repository ${env.BRANCH} branch from ${fromDateString} to ${toDateString}`,
   );
   const blocks: AnyBlock[] = [
     {
@@ -152,7 +156,7 @@ async function main() {
             },
             {
               type: 'text',
-              text: ' Failed tests on the ',
+              text: ' Top 10 failed tests on the ',
             },
             {
               type: 'link',
@@ -170,7 +174,7 @@ async function main() {
             },
             {
               type: 'text',
-              text: ` branch from ${fromDateString} to ${toDateString} on GitHub Actions`,
+              text: ` branch from ${fromDateString} to ${toDateString}`,
             },
           ],
         },

.github/scripts/shared/test-reports.ts

@@ -17,8 +17,9 @@ export interface TestSuite {
   name: string;
   job: {
     name: string;
-    id: string;
-    runId: string;
+    id: number;
+    runId: number;
+    prNumber: number;
   };
   date: Date;
   tests: number;

.github/workflows/e2e-chrome.yml

@@ -81,14 +81,13 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ !cancelled() }}
     env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       OWNER: ${{ github.repository_owner }}
       REPOSITORY: ${{ github.event.repository.name }}
       # For a `pull_request` event, the branch is `github.head_ref`.
       # For a `push` event, the branch is `github.ref_name`.
       BRANCH: ${{ github.head_ref || github.ref_name }}
-      TEST_RESULTS_PATH: test/test-results/e2e
-      RUN_ID: ${{ github.run_id }}
-      PR_NUMBER: ${{ github.event.pull_request.number }}
+      TEST_RUNS_PATH: test/test-results/test-runs-chrome.json
     steps:
       - name: Checkout and setup environment
         uses: MetaMask/action-checkout-and-setup@v1
@@ -110,4 +109,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: test-e2e-chrome-report
-          path: test/test-results/test-runs.json
+          path: ${{ env.TEST_RUNS_PATH }}

.github/workflows/e2e-firefox.yml

@@ -38,14 +38,13 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ !cancelled() }}
     env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       OWNER: ${{ github.repository_owner }}
       REPOSITORY: ${{ github.event.repository.name }}
       # For a `pull_request` event, the branch is `github.head_ref`.
       # For a `push` event, the branch is `github.ref_name`.
       BRANCH: ${{ github.head_ref || github.ref_name }}
-      TEST_RESULTS_PATH: test/test-results/e2e
-      RUN_ID: ${{ github.run_id }}
-      PR_NUMBER: ${{ github.event.pull_request.number }}
+      TEST_RUNS_PATH: test/test-results/test-runs-firefox.json
     steps:
       - name: Checkout and setup environment
         uses: MetaMask/action-checkout-and-setup@v1
@@ -67,4 +66,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: test-e2e-firefox-report
-          path: test/test-results/test-runs.json
+          path: ${{ env.TEST_RUNS_PATH }}

.github/workflows/run-e2e.yml

@@ -45,11 +45,9 @@ jobs:
       BRANCH: ${{ github.head_ref || github.ref_name }}
       MATRIX_INDEX: ${{ inputs.matrix-index }}
       MATRIX_TOTAL: ${{ inputs.matrix-total }}
-      OWNER: ${{ github.repository_owner }}
-      REPOSITORY: ${{ github.event.repository.name }}
-      RUN_ID: ${{ github.run_id }}
       JOB_NAME: ${{ inputs.test-suite-name }}${{ inputs.matrix-total > 1 && format(' ({0})', inputs.matrix-index) || '' }}
-      ATTEMPT_NUMBER: ${{ github.run_attempt }}
+      RUN_ID: ${{ github.run_id }}
+      PR_NUMBER: ${{ github.event.pull_request.number || '' }}
     steps:
       - name: Install dependencies
         run: |
@@ -95,16 +93,12 @@ jobs:
       - run: yarn tsx .github/scripts/git-diff-default-branch.ts
         if: ${{ steps.download-changed-files.outcome == 'failure' }}
 
-      - name: Get job id
-        id: get-job-id
-        run: yarn tsx .github/scripts/get-job-id.ts
-
       - name: Run E2E tests
         timeout-minutes: ${{ inputs.test-timeout-minutes }}
         run: ${{ inputs.test-command }} --retries 1
         env:
           # The properties are picked up by mocha-junit-reporter and added to the XML test results
-          PROPERTIES: 'JOB_NAME:${{ env.JOB_NAME }},JOB_ID:${{ steps.get-job-id.outputs.job-id }},RUN_ID:${{ env.RUN_ID }}'
+          PROPERTIES: 'JOB_NAME:${{ env.JOB_NAME }},RUN_ID:${{ env.RUN_ID }},PR_NUMBER:${{ env.PR_NUMBER }}'
 
       - name: Upload test results and artifacts
         if: ${{ !cancelled() }}

package.json

@@ -462,6 +462,7 @@
     "@metamask/test-dapp-multichain": "^0.10.0",
     "@metamask/test-dapp-solana": "^0.1.0",
     "@octokit/core": "^3.6.0",
+    "@octokit/types": "^14.0.0",
     "@open-rpc/meta-schema": "^1.14.6",
     "@open-rpc/mock-server": "^1.7.5",
     "@open-rpc/schema-utils-js": "^2.0.5",

yarn.lock

@@ -30104,6 +30104,7 @@ __metadata:
     "@ngraveio/bc-ur": "npm:^1.1.13"
     "@noble/hashes": "npm:^1.3.3"
     "@octokit/core": "npm:^3.6.0"
+    "@octokit/types": "npm:^14.0.0"
     "@open-rpc/meta-schema": "npm:^1.14.6"
     "@open-rpc/mock-server": "npm:^1.7.5"
     "@open-rpc/schema-utils-js": "npm:^2.0.5"