ci: DEV-2482: E2E playwright (#648)

* DEV-2482: Migrate to Playwright

* Enable action

* Fix workflow

* Fix run command

* Run with workers

* FIx yarn cache

* Update Codecept config

* Enable codecoverage

* Fix timeout

* Parallelize "Images' labels type matching" test

* Use suites for tests execution

* Increase actions timeout

* Split modules instalation and e2e

* Yarn cache e2e

* Add Istanbul converter for the code coverage

* Fix tests

* Increace timeouts

* Fix NER test

* Update build flow

* Fix image test

* Fix cache key

* Restart browser after each test
Decrease number of workers to match CPU cores

* Rename coverage converter

* Gather CPU cores number

* Fix coverage upload

* DIsable coverage collection

* Update codecept config

* Disable coverage upload

* Freeze lockfile for yarn installation

* Remove unnecessary apuse

* remove pause from globals

Co-authored-by: hlomzik <[email protected]>

Author: nick-skriabin

.github/workflows/build.yml

@@ -21,6 +21,10 @@ jobs:
       with:
         node-version: "${{ env.NODE }}"
 
+    - name: Get CPU info
+      id: "cpu-info"
+      run: echo "::set-output name=cores-count::$(cat /proc/cpuinfo  | grep processor | wc -l)"
+
     - name: Upgrade Yarn
       run: npm install -g [email protected]
 
@@ -32,16 +36,18 @@ jobs:
       uses: actions/cache@v3
       with:
         path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ env.CACHE_NAME_PREFIX }}-${{ runner.os }}-node-${{ env.NODE }}-${{ hashFiles('**/package.json') }}-${{ hashFiles('**/yarn.lock') }}
+        key: ${{ env.CACHE_NAME_PREFIX }}-${{ runner.os }}-node-${{ env.NODE }}-${{ hashFiles('**/yarn.lock') }}
+        restore-keys: |
+          yarn-${{ env.CACHE_NAME_PREFIX }}-${{ runner.os }}-node-${{ env.NODE }}-
 
     - name: Print Yarn cache size
       run: du -d 0 -h ${{ steps.yarn-cache-dir-path.outputs.dir }}
 
     - name: Install dependencies
-      run: rm package-lock.json && yarn install
+      run: yarn install --frozen-lockfile
 
     - name: Unit tests
-      run: yarn test
+      run: yarn test && yarn test:coverage
 
     - name: Build distribution package
       timeout-minutes: 10
@@ -53,17 +59,49 @@ jobs:
     # run http-server with build in background (will be killed after job ends)
     # do this only for master branch (so only for push event)
     # because pr can contain unfinished job
-    - run: npx serve -l tcp://localhost:3000 build &
+    - name: "Run server"
+      run: npx serve -l tcp://localhost:3000 build &
       if: github.event_name == 'push'
+
     - id: wait_for_npx_server
+      name: "Wait for server"
       timeout-minutes: 1
       run: |
         while [ "$(curl -s -o /dev/null -L -w ''%{http_code}'' "http://localhost:3000/")" != "200" ]; do
           echo "=> Waiting for service to become available" && sleep 2s
         done
-    - run: yarn run test:e2e:headless
-      timeout-minutes: 15
+
+    - name: "Setup e2e"
+      if: github.event_name == 'push'
+      timeout-minutes: 1
+      run: |
+        set -euo pipefail
+        cd e2e
+        yarn install --frozen-lockfile
+
+    - name: Run e2e test suite
       if: github.event_name == 'push'
+      timeout-minutes: 30
+      run: |
+        set -euo pipefail
+        cd e2e
+        yarn run test:ci ${{ steps.cpu-info.outputs.cores-count }}
+
+    # - name: "Convert coverage report to Istanbul"
+    #   if: github.event_name == 'push'
+    #   run: |
+    #     set -euo pipefail
+    #     cd e2e
+    #     yarn run coverage:istanbul
+    #     yarn run coverage:report
+
+    # - name: "Upload e2e coverage to Codecov"
+    #   uses: codecov/[email protected]
+    #   with:
+    #     name: codecov-general
+    #     directory: ./e2e/output/coverage
+    #     token: ${{ secrets.CODECOV_TOKEN }}
+    #     fail_ci_if_error: true
 
     - uses: actions/upload-artifact@v3
       if: ${{ failure() }}

codecov.yml

@@ -0,0 +1,2 @@
+fixes:
+  - "::src/"

e2e/codecept.conf.js

@@ -2,28 +2,28 @@
 // HEADLESS=true npx codecept run
 const headless = process.env.HEADLESS;
 
-// codecept can make screenshot on every step and create html
-// with "gif" and annotations. possible usage:
-// GIF=true yarn e2e:test:headless e2e/present_feature_test.js
-const recordVideo = process.env.GIF
-  ? {
-    stepByStepReport: {
-      enabled: true,
-      deleteSuccessful: false,
-    },
-  }
-  : null;
-
-// eslint-disable-next-line no-undef
-exports.config = {
-  tests: "./**/*.test.js",
+module.exports.config = {
+  timeout: 60 * 30, // Time out after 30 minutes
+  tests: "./tests/*.test.js",
   output: "./output",
   helpers: {
-    Puppeteer: {
+    // Puppeteer: {
+    //   url: "http://localhost:3000",
+    //   show: !headless,
+    //   waitForAction: headless ? 100 : 1200,
+    //   windowSize: "1200x900",
+    // },
+    Playwright: {
       url: "http://localhost:3000",
       show: !headless,
+      restart: 'context',
+      timeout: 60000, // Action timeout after 60 seconds
       waitForAction: headless ? 100 : 1200,
       windowSize: "1200x900",
+      waitForNavigation: "networkidle",
+      browser: "chromium",
+      trace: false,
+      keepTraceForPassedTests: false,
     },
     MouseActions: {
       require: "./helpers/MouseActions.js",
@@ -45,15 +45,34 @@ exports.config = {
     ErrorsCollector: "./fragments/ErrorsCollector.js",
   },
   bootstrap: null,
-  mocha: {},
+  mocha: {
+    bail: true,
+    reporterOptions: {
+      mochaFile: "output/result.xml",
+    },
+  },
   name: "label-studio-frontend",
   plugins: {
     retryFailedStep: {
       enabled: true,
+      minTimeout: 100,
+      defaultIgnoredSteps: [
+        //'amOnPage',
+        //'wait*',
+        'send*',
+        'execute*',
+        'run*',
+        'have*',
+      ],
     },
+    // For the future generations
+    // coverage: {
+    //   enabled: true,
+    //   coverageDir: "output/coverage",
+    // },
     screenshotOnFail: {
       enabled: true,
     },
-    ...recordVideo,
   },
+  require: ['ts-node/register'],
 };

e2e/coverage-to-istanbul.js

@@ -0,0 +1,38 @@
+const fs = require('fs/promises');
+const path = require('path');
+
+const v8toIstanbul = require('v8-to-istanbul');
+const covDir = './output/coverage';
+
+const convertCoverage = async (fileName) => {
+  if (fileName.match('istanbul')) return;
+
+  const coverage = require(`${covDir}/${fileName}`);
+  const basename = path.basename(fileName).replace('.coverage.json', '');
+  const finalName = `${covDir}/${basename}_final.coverage.json`;
+
+  for (const entry of coverage) {
+    // Used to get file name
+    const file = entry.url.match(/(?:http(s)*:\/\/.*\/)(?<file>.*)/);
+    const converter = new v8toIstanbul(file.groups.file, 0, {
+      source: entry.source,
+    });
+
+    await converter.load();
+    converter.applyCoverage(entry.functions);
+
+    // Store converted coverage file which can later be used to generate report
+    await fs.writeFile(
+      finalName,
+      JSON.stringify(converter.toIstanbul(), null, 2),
+    );
+    console.log(`Processed ${basename}`);
+  }
+
+  await fs.unlink(`${covDir}/${fileName}`);
+};
+
+// read all the coverage file from output/coverage folder
+fs.readdir(covDir).then(files => {
+  files.forEach(convertCoverage);
+});

e2e/fragments/AtAudioView.js

@@ -5,6 +5,6 @@ const Helpers = require("../tests/helpers");
 
 module.exports = {
   waitForAudio() {
-    I.executeAsyncScript(Helpers.waitForAudio);
+    I.executeScript(Helpers.waitForAudio);
   },
 };

e2e/fragments/AtImageView.js

@@ -21,30 +21,31 @@ module.exports = {
   },
 
   waitForImage() {
-    I.executeAsyncScript(Helpers.waitForImage);
+    I.say("Waiting for image to be loaded");
+    I.executeScript(Helpers.waitForImage);
     I.waitForVisible("canvas", 5);
   },
 
   async getCanvasSize() {
-    const sizes = await I.executeAsyncScript(Helpers.getCanvasSize);
+    const sizes = await I.executeScript(Helpers.getCanvasSize);
 
     return sizes;
   },
 
   async getImageSize() {
-    const sizes = await I.executeAsyncScript(Helpers.getImageSize);
+    const sizes = await I.executeScript(Helpers.getImageSize);
 
     return sizes;
   },
 
   async getImageFrameSize() {
-    const sizes = await I.executeAsyncScript(Helpers.getImageFrameSize);
+    const sizes = await I.executeScript(Helpers.getImageFrameSize);
 
     return sizes;
   },
 
   setZoom(scale, x, y) {
-    I.executeAsyncScript(Helpers.setZoom, scale, x, y);
+    I.executeScript(Helpers.setZoom, [scale, x, y]);
   },
 
   /**
@@ -53,22 +54,22 @@ module.exports = {
    * @param {number} y
    */
   clickKonva(x, y) {
-    I.executeAsyncScript(Helpers.clickKonva, x, y);
+    I.executeScript(Helpers.clickKonva, [x, y]);
   },
   /**
    * Click multiple times on the main Stage
    * @param {number[][]} points
    */
   clickPointsKonva(points) {
-    I.executeAsyncScript(Helpers.clickMultipleKonva, points);
+    I.executeScript(Helpers.clickMultipleKonva, points);
   },
   /**
    * Click multiple times on the main Stage then close Polygon
    * @param {number[][]} points
    * @deprecated Use drawByClickingPoints instead
    */
   clickPolygonPointsKonva(points) {
-    I.executeAsyncScript(Helpers.polygonKonva, points);
+    I.executeScript(Helpers.polygonKonva, points);
   },
   /**
    * Dragging between two points
@@ -79,7 +80,7 @@ module.exports = {
    * @deprecated Use drawByDrag instead
    */
   dragKonva(x, y, shiftX, shiftY) {
-    I.executeAsyncScript(Helpers.dragKonva, x, y, shiftX, shiftY);
+    I.executeScript(Helpers.dragKonva, [x, y, shiftX, shiftY]);
   },
 
   /**
@@ -90,33 +91,33 @@ module.exports = {
    * @param {number} tolerance
    */
   async hasPixelColor(x, y, rgbArray, tolerance = 3) {
-    const colorPixels = await I.executeAsyncScript(Helpers.getKonvaPixelColorFromPoint, x, y);
+    const colorPixels = await I.executeScript(Helpers.getKonvaPixelColorFromPoint, [x, y]);
     const hasPixel = Helpers.areEqualRGB(rgbArray, colorPixels, tolerance);
 
     return hasPixel;
   },
 
   // Only for debugging
   async whereIsPixel(rgbArray, tolerance = 3) {
-    const points = await I.executeAsyncScript(Helpers.whereIsPixel, rgbArray, tolerance);
+    const points = await I.executeScript(Helpers.whereIsPixel, [rgbArray, tolerance]);
 
     return points;
   },
 
   async countKonvaShapes() {
-    const count = await I.executeAsyncScript(Helpers.countKonvaShapes);
+    const count = await I.executeScript(Helpers.countKonvaShapes);
 
     return count;
   },
 
   async isTransformerExist() {
-    const isTransformerExist = await I.executeAsyncScript(Helpers.isTransformerExist);
+    const isTransformerExist = await I.executeScript(Helpers.isTransformerExist);
 
     return isTransformerExist;
   },
 
   async isRotaterExist() {
-    const isRotaterExist = await I.executeAsyncScript(Helpers.isRotaterExist);
+    const isRotaterExist = await I.executeScript(Helpers.isRotaterExist);
 
     return isRotaterExist;
   },
@@ -190,6 +191,7 @@ module.exports = {
   clickAt(x, y) {
     I.scrollPageToTop();
     I.clickAt(this._stageBBox.x + x, this._stageBBox.y + y);
+    I.wait(1); // We gotta  wait here because clicks on the canvas are not processed immediately
   },
   dblClickAt(x, y) {
     I.scrollPageToTop();

e2e/fragments/AtRichText.js

@@ -6,7 +6,7 @@ const Helpers = require("../tests/helpers");
 module.exports = {
   _rootSelector: ".lsf-htx-richtext",
   selectTextByGlobalOffset(startOffset, endOffset) {
-    I.executeAsyncScript(Helpers.selectText, {
+    I.executeScript(Helpers.selectText, {
       selector: ".lsf-htx-richtext",
       rangeStart: startOffset,
       rangeEnd: endOffset,

e2e/fragments/AtSidebar.js

@@ -1,3 +1,5 @@
+const { assert } = require("assert");
+
 /* global inject, locate */
 const { I } = inject();
 
@@ -9,16 +11,16 @@ module.exports = {
   _selectedRegionsLocator: locate(".lsf-entity"),
   seeRegions(count) {
     if (count) {
-      I.see(`Regions\n\u00A0${count}`, this._sideBarLocator);
+      I.seeElement(this._regionsCounterLocator.withText(`${count}`));
     } else {
       I.seeElement(this._regionGroupButton.withText("Regions"));
       I.dontSeeElement(this._regionGroupButton.withDescendant(this._regionsCounterLocator));
     }
   },
   dontSeeRegions(count) {
     if (count) {
-      I.dontSee(`Regions\n\u00A0${count}`, this._sideBarLocator);
-    } else if (count===+count) {
+      I.dontSeeElement(this._regionsCounterLocator.withText(`${count}`));
+    } else if (count === +count) {
       I.seeElement(this._regionGroupButton.withDescendant(this._regionsCounterLocator));
     } else {
       I.dontSee("Regions", this._sideBarLocator);
@@ -51,4 +53,7 @@ module.exports = {
   clickRegion(text) {
     I.click(this._regionLocator.withText(`${text}`));
   },
+  selectTool(tool) {
+    I.click(`[aria-label=${tool}-tool]`);
+  },
 };