[test-optimization] [SDTEST-2163] Check framework version in capabilities tagging

integration-tests/playwright/playwright.spec.js

@@ -1363,12 +1363,20 @@ versions.forEach((version) => {
             assert.isNotEmpty(metadataDicts)
             metadataDicts.forEach(metadata => {
               assert.equal(metadata.test[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS], undefined)
-              assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], '1')
               assert.equal(metadata.test[DD_CAPABILITIES_AUTO_TEST_RETRIES], '1')
-              assert.equal(metadata.test[DD_CAPABILITIES_IMPACTED_TESTS], '1')
-              assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE], '1')
-              assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE], '1')
-              assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX], '4')
+              if (satisfies(version, '>=1.38.0') || version === 'latest') {
+                assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], '1')
+                assert.equal(metadata.test[DD_CAPABILITIES_IMPACTED_TESTS], '1')
+                assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE], '1')
+                assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE], '1')
+                assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX], '4')
+              } else {
+                assert.equal(metadata.test[DD_CAPABILITIES_EARLY_FLAKE_DETECTION], undefined)
+                assert.equal(metadata.test[DD_CAPABILITIES_IMPACTED_TESTS], undefined)
+                assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE], undefined)
+                assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE], undefined)
+                assert.equal(metadata.test[DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX], undefined)
+              }
               // capabilities logic does not overwrite test session name
               assert.equal(metadata.test[TEST_SESSION_NAME], 'my-test-session-name')
             })

packages/datadog-instrumentations/src/cucumber.js

@@ -157,9 +157,9 @@ function getErrorFromCucumberResult (cucumberResult) {
   return error
 }
 
-function getChannelPromise (channelToPublishTo, isParallel = false) {
+function getChannelPromise (channelToPublishTo, isParallel = false, frameworkVersion = null) {
   return new Promise(resolve => {
-    channelToPublishTo.publish({ onDone: resolve, isParallel })
+    channelToPublishTo.publish({ onDone: resolve, isParallel, frameworkVersion })
   })
 }
 
@@ -451,7 +451,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin
     }
     let errorSkippableRequest
 
-    const configurationResponse = await getChannelPromise(libraryConfigurationCh, isParallel)
+    const configurationResponse = await getChannelPromise(libraryConfigurationCh, isParallel, frameworkVersion)
 
     isEarlyFlakeDetectionEnabled = configurationResponse.libraryConfig?.isEarlyFlakeDetectionEnabled
     earlyFlakeDetectionNumRetries = configurationResponse.libraryConfig?.earlyFlakeDetectionNumRetries

packages/datadog-instrumentations/src/jest.js

@@ -726,7 +726,7 @@ function getCliWrapper (isNewJestVersion) {
         return runCLI.apply(this, arguments)
       }
 
-      libraryConfigurationCh.publish({ onDone })
+      libraryConfigurationCh.publish({ onDone, frameworkVersion: jestVersion })
 
       try {
         const { err, libraryConfig } = await configurationPromise

packages/datadog-instrumentations/src/mocha/main.js

@@ -201,9 +201,10 @@ function getOnEndHandler (isParallel) {
   }
 }
 
-function getExecutionConfiguration (runner, isParallel, onFinishRequest) {
+function getExecutionConfiguration (runner, isParallel, frameworkVersion, onFinishRequest) {
   const ctx = {
-    isParallel
+    isParallel,
+    frameworkVersion
   }
 
   const onReceivedSkippableSuites = ({ err, skippableSuites, itrCorrelationId: responseItrCorrelationId }) => {
@@ -343,7 +344,7 @@ addHook({
   name: 'mocha',
   versions: ['>=5.2.0'],
   file: 'lib/mocha.js'
-}, (Mocha) => {
+}, (Mocha, frameworkVersion) => {
   shimmer.wrap(Mocha.prototype, 'run', run => function () {
     // Workers do not need to request any data, just run the tests
     if (!testFinishCh.hasSubscribers || getEnvironmentVariable('MOCHA_WORKER_ID') || this.options.parallel) {
@@ -363,7 +364,7 @@ addHook({
       }
     })
 
-    getExecutionConfiguration(runner, false, () => {
+    getExecutionConfiguration(runner, false, frameworkVersion, () => {
       if (config.isKnownTestsEnabled) {
         const testSuites = this.files.map(file => getTestSuitePath(file, process.cwd()))
         const isFaulty = getIsFaultyEarlyFlakeDetection(
@@ -616,7 +617,7 @@ addHook({
     this.once('start', getOnStartHandler(true, frameworkVersion))
     this.once('end', getOnEndHandler(true))
 
-    getExecutionConfiguration(this, true, () => {
+    getExecutionConfiguration(this, true, frameworkVersion, () => {
       if (config.isKnownTestsEnabled) {
         const testSuites = files.map(file => getTestSuitePath(file, process.cwd()))
         const isFaulty = getIsFaultyEarlyFlakeDetection(

packages/datadog-instrumentations/src/playwright.js

@@ -516,7 +516,10 @@ function runnerHook (runnerExport, playwrightVersion) {
     testSessionStartCh.publish({ command, frameworkVersion: playwrightVersion, rootDir })
 
     try {
-      const { err, libraryConfig } = await getChannelPromise(libraryConfigurationCh)
+      const { err, libraryConfig } = await getChannelPromise(
+        libraryConfigurationCh,
+        { frameworkVersion: playwrightVersion }
+      )
       if (!err) {
         isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled
         isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled

packages/datadog-instrumentations/src/vitest.js

@@ -113,9 +113,9 @@ function isBaseSequencer (vitestPackage) {
   return vitestPackage.b?.name === 'BaseSequencer'
 }
 
-function getChannelPromise (channelToPublishTo) {
+function getChannelPromise (channelToPublishTo, frameworkVersion) {
   return new Promise(resolve => {
-    channelToPublishTo.publish({ onDone: resolve })
+    channelToPublishTo.publish({ onDone: resolve, frameworkVersion })
   })
 }
 
@@ -173,7 +173,7 @@ function getTestName (task) {
   return testName
 }
 
-function getSortWrapper (sort) {
+function getSortWrapper (sort, frameworkVersion) {
   return async function () {
     if (!testSessionFinishCh.hasSubscribers) {
       return sort.apply(this, arguments)
@@ -193,7 +193,7 @@ function getSortWrapper (sort) {
     let isDiEnabled = false
 
     try {
-      const { err, libraryConfig } = await getChannelPromise(libraryConfigurationCh)
+      const { err, libraryConfig } = await getChannelPromise(libraryConfigurationCh, frameworkVersion)
       if (!err) {
         isFlakyTestRetriesEnabled = libraryConfig.isFlakyTestRetriesEnabled
         flakyTestRetriesCount = libraryConfig.flakyTestRetriesCount

packages/datadog-plugin-cypress/src/cypress-plugin.js

@@ -550,7 +550,7 @@ class CypressPlugin {
           [TEST_SESSION_NAME]: testSessionName
         }
       }
-      const libraryCapabilitiesTags = getLibraryCapabilitiesTags(this.constructor.id)
+      const libraryCapabilitiesTags = getLibraryCapabilitiesTags(this.constructor.id, false, this.frameworkVersion)
       metadataTags.test = {
         ...metadataTags.test,
         ...libraryCapabilitiesTags

packages/dd-trace/src/plugins/ci_plugin.js

@@ -75,7 +75,7 @@ module.exports = class CiPlugin extends Plugin {
     this.rootDir = process.cwd() // fallback in case :session:start events are not emitted
 
     this.addSub(`ci:${this.constructor.id}:library-configuration`, (ctx) => {
-      const { onDone, isParallel } = ctx
+      const { onDone, isParallel, frameworkVersion } = ctx
       ctx.currentStore = storage('legacy').getStore()
 
       if (!this.tracer._exporter || !this.tracer._exporter.getLibraryConfiguration) {
@@ -88,7 +88,7 @@ module.exports = class CiPlugin extends Plugin {
           this.libraryConfig = libraryConfig
         }
 
-        const libraryCapabilitiesTags = getLibraryCapabilitiesTags(this.constructor.id, isParallel)
+        const libraryCapabilitiesTags = getLibraryCapabilitiesTags(this.constructor.id, isParallel, frameworkVersion)
         const metadataTags = {
           test: {
             ...libraryCapabilitiesTags

packages/dd-trace/src/plugins/util/test.js

@@ -3,6 +3,7 @@ const fs = require('fs')
 const { URL } = require('url')
 const log = require('../../log')
 const { getEnvironmentVariable } = require('../../config-helper')
+const satisfies = require('semifies')
 
 const istanbul = require('istanbul-lib-coverage')
 const ignore = require('ignore')
@@ -129,6 +130,18 @@ const DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE = '_dd.library_capabilities.test_m
 const DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX = '_dd.library_capabilities.test_management.attempt_to_fix'
 const UNSUPPORTED_TIA_FRAMEWORKS = new Set(['playwright', 'vitest'])
 const UNSUPPORTED_TIA_FRAMEWORKS_PARALLEL_MODE = new Set(['cucumber', 'mocha'])
+const MINIMUM_FRAMEWORK_VERSION_FOR_EFD = {
+  playwright: '>=1.38.0'
+}
+const MINIMUM_FRAMEWORK_VERSION_FOR_IMPACTED_TESTS = {
+  playwright: '>=1.38.0'
+}
+const MINIMUM_FRAMEWORK_VERSION_FOR_QUARANTINE = {
+  playwright: '>=1.38.0'
+}
+const MINIMUM_FRAMEWORK_VERSION_FOR_DISABLE = {
+  playwright: '>=1.38.0'
+}
 const UNSUPPORTED_ATTEMPT_TO_FIX_FRAMEWORKS_PARALLEL_MODE = new Set(['mocha'])
 const NOT_SUPPORTED_GRANULARITY_IMPACTED_TESTS_FRAMEWORKS = new Set(['mocha', 'playwright', 'vitest'])
 
@@ -895,18 +908,52 @@ function isTiaSupported (testFramework, isParallel) {
            (isParallel && UNSUPPORTED_TIA_FRAMEWORKS_PARALLEL_MODE.has(testFramework)))
 }
 
+function isEarlyFlakeDetectionSupported (testFramework, frameworkVersion) {
+  return testFramework === 'playwright'
+    ? satisfies(frameworkVersion, MINIMUM_FRAMEWORK_VERSION_FOR_EFD[testFramework])
+    : true
+}
+
+function isImpactedTestsSupported (testFramework, frameworkVersion) {
+  return testFramework === 'playwright'
+    ? satisfies(frameworkVersion, MINIMUM_FRAMEWORK_VERSION_FOR_IMPACTED_TESTS[testFramework])
+    : true
+}
+
+function isQuarantineSupported (testFramework, frameworkVersion) {
+  return testFramework === 'playwright'
+    ? satisfies(frameworkVersion, MINIMUM_FRAMEWORK_VERSION_FOR_QUARANTINE[testFramework])
+    : true
+}
+
+function isDisableSupported (testFramework, frameworkVersion) {
+  return testFramework === 'playwright'
+    ? satisfies(frameworkVersion, MINIMUM_FRAMEWORK_VERSION_FOR_DISABLE[testFramework])
+    : true
+}
+
 function isAttemptToFixSupported (testFramework, isParallel) {
   return !(isParallel && UNSUPPORTED_ATTEMPT_TO_FIX_FRAMEWORKS_PARALLEL_MODE.has(testFramework))
 }
 
-function getLibraryCapabilitiesTags (testFramework, isParallel) {
+function getLibraryCapabilitiesTags (testFramework, isParallel, frameworkVersion) {
   return {
-    [DD_CAPABILITIES_TEST_IMPACT_ANALYSIS]: isTiaSupported(testFramework, isParallel) ? '1' : undefined,
-    [DD_CAPABILITIES_EARLY_FLAKE_DETECTION]: '1',
+    [DD_CAPABILITIES_TEST_IMPACT_ANALYSIS]: isTiaSupported(testFramework, isParallel)
+      ? '1'
+      : undefined,
+    [DD_CAPABILITIES_EARLY_FLAKE_DETECTION]: isEarlyFlakeDetectionSupported(testFramework, frameworkVersion)
+      ? '1'
+      : undefined,
     [DD_CAPABILITIES_AUTO_TEST_RETRIES]: '1',
-    [DD_CAPABILITIES_IMPACTED_TESTS]: '1',
-    [DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE]: '1',
-    [DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE]: '1',
+    [DD_CAPABILITIES_IMPACTED_TESTS]: isImpactedTestsSupported(testFramework, frameworkVersion)
+      ? '1'
+      : undefined,
+    [DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE]: isQuarantineSupported(testFramework, frameworkVersion)
+      ? '1'
+      : undefined,
+    [DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE]: isDisableSupported(testFramework, frameworkVersion)
+      ? '1'
+      : undefined,
     [DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX]: isAttemptToFixSupported(testFramework, isParallel)
       ? '4'
       : undefined