Set up crash-tracking JVM flags automatically (for Hotspot)

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -198,6 +198,11 @@ public static void start(
       injectAgentArgsConfig(agentArgs);
     }
 
+    // We want to have this enabled as soon as possible
+    // It entails loading a native library so let's see if it works without regression in startup
+    // time
+    initializeErrorTracking();
+
     // Retro-compatibility for the old way to configure CI Visibility
     if ("true".equals(ddGetProperty("dd.integration.junit.enabled"))
         || "true".equals(ddGetProperty("dd.integration.testng.enabled"))) {
@@ -759,8 +764,6 @@ private static synchronized void startJmx() {
     if (jmxStarting.getAndSet(true)) {
       return; // another thread is already in startJmx
     }
-    // error tracking initialization relies on JMX being available
-    initializeErrorTracking();
     if (jmxFetchEnabled) {
       startJmxFetch();
     }
@@ -1025,7 +1028,7 @@ private static void initializeErrorTracking() {
       return;
     }
     try {
-      Class<?> clz = AGENT_CLASSLOADER.loadClass("com.datadog.crashtracking.ScriptInitializer");
+      Class<?> clz = AGENT_CLASSLOADER.loadClass("com.datadog.crashtracking.Initializer");
       clz.getMethod("initialize").invoke(null);
     } catch (Throwable t) {
       log.debug("Unable to initialize crash uploader", t);

dd-java-agent/agent-crashtracking/build.gradle

@@ -17,6 +17,8 @@ dependencies {
   implementation project(':utils:container-utils')
   implementation project(':utils:version-utils')
 
+  implementation project.hasProperty('ddprof.jar') ? files(project.getProperty('ddprof.jar')) : libs.ddprof
+
   implementation libs.okhttp
   implementation libs.moshi
 

dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashUploaderScriptInitializer.java

@@ -1,11 +1,11 @@
 package com.datadog.crashtracking;
 
-import static com.datadog.crashtracking.ScriptInitializer.LOG;
-import static com.datadog.crashtracking.ScriptInitializer.RWXRWXRWX;
-import static com.datadog.crashtracking.ScriptInitializer.R_XR_XR_X;
-import static com.datadog.crashtracking.ScriptInitializer.findAgentJar;
-import static com.datadog.crashtracking.ScriptInitializer.getCrashUploaderTemplate;
-import static com.datadog.crashtracking.ScriptInitializer.writeConfig;
+import static com.datadog.crashtracking.Initializer.LOG;
+import static com.datadog.crashtracking.Initializer.RWXRWXRWX;
+import static com.datadog.crashtracking.Initializer.R_XR_XR_X;
+import static com.datadog.crashtracking.Initializer.findAgentJar;
+import static com.datadog.crashtracking.Initializer.getCrashUploaderTemplate;
+import static com.datadog.crashtracking.Initializer.writeConfig;
 import static java.nio.file.attribute.PosixFilePermissions.asFileAttribute;
 import static java.nio.file.attribute.PosixFilePermissions.fromString;
 import static java.util.Locale.ROOT;

dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/ScriptInitializer.java → dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/Initializer.java

@@ -4,13 +4,14 @@
 import static java.util.Comparator.reverseOrder;
 import static java.util.Locale.ROOT;
 
-import com.sun.management.HotSpotDiagnosticMXBean;
+import com.datadoghq.profiler.JVMAccess;
 import datadog.trace.api.Platform;
+import datadog.trace.api.config.ProfilingConfig;
+import datadog.trace.bootstrap.config.provider.ConfigProvider;
 import datadog.trace.util.PidHelper;
 import java.io.BufferedWriter;
 import java.io.IOException;
 import java.io.InputStream;
-import java.lang.management.ManagementFactory;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -20,19 +21,29 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public final class ScriptInitializer {
-  static final Logger LOG = LoggerFactory.getLogger(ScriptInitializer.class);
+public final class Initializer {
+  static final Logger LOG = LoggerFactory.getLogger(Initializer.class);
   static final String PID_PREFIX = "_pid";
   static final String RWXRWXRWX = "rwxrwxrwx";
   static final String R_XR_XR_X = "r-xr-xr-x";
 
   public static void initialize() {
-    // this is HotSpot specific implementation (eg. will not work for IBM J9)
-    HotSpotDiagnosticMXBean diagBean =
-        ManagementFactory.getPlatformMXBean(HotSpotDiagnosticMXBean.class);
+    ConfigProvider cfgProvider = ConfigProvider.getInstance();
+    String scratchDir = cfgProvider.getString(ProfilingConfig.PROFILING_DATADOG_PROFILER_SCRATCH);
 
-    initializeCrashUploader(diagBean);
-    initializeOOMENotifier(diagBean);
+    JVMAccess jvmAccess =
+        new JVMAccess(
+            null,
+            scratchDir,
+            throwable -> {
+              logInitializationError(
+                  "Unexpected exception while initializing JVMAccess", throwable);
+            });
+
+    JVMAccess.Flags flags = jvmAccess.flags();
+
+    initializeCrashUploader(flags);
+    initializeOOMENotifier(flags);
   }
 
   static InputStream getCrashUploaderTemplate() {
@@ -130,28 +141,75 @@ private static String getBaseName(Path path) {
    * `dd_crash_uploader.bat` and the script does not exist it will be created and prefilled with
    * code ensuring the error log upload will be triggered on JVM crash.
    */
-  private static void initializeCrashUploader(HotSpotDiagnosticMXBean diagBean) {
+  private static void initializeCrashUploader(JVMAccess.Flags flags) {
     try {
-      String onErrorVal = diagBean.getVMOption("OnError").getValue();
-      String onErrorFile = diagBean.getVMOption("ErrorFile").getValue();
-      CrashUploaderScriptInitializer.initialize(onErrorVal, onErrorFile);
+      String onErrorVal = flags.getStringFlag("OnError");
+      String onErrorFile = flags.getStringFlag("ErrorFile");
+
+      String uploadScript = getScript("dd_crash_uploader");
+      if (onErrorVal == null || onErrorVal.isEmpty()) {
+        onErrorVal = uploadScript;
+      } else if (!onErrorVal.contains("dd_crash_uploader")) {
+        // we can chain scripts so let's preserve the original value in addition to our crash
+        // uploader
+        onErrorVal = uploadScript + "; " + onErrorVal;
+      }
+
+      // set the JVM flag
+      flags.setStringFlag("OnError", onErrorVal);
+      if (LOG.isDebugEnabled()) {
+        String currentVal = flags.getStringFlag("OnError");
+        if (!currentVal.equals(uploadScript)) {
+          LOG.debug("Unable to set OnError flag to {}. Crash-tracking may not work.", currentVal);
+        }
+      }
+
+      CrashUploaderScriptInitializer.initialize(uploadScript, onErrorFile);
     } catch (Throwable t) {
       logInitializationError(
           "Unexpected exception while creating custom crash upload script. Crash tracking will not work properly.",
           t);
     }
   }
 
-  private static void initializeOOMENotifier(HotSpotDiagnosticMXBean diagBean) {
+  private static void initializeOOMENotifier(JVMAccess.Flags flags) {
     try {
-      String onOutOfMemoryVal = diagBean.getVMOption("OnOutOfMemoryError").getValue();
-      OOMENotifierScriptInitializer.initialize(onOutOfMemoryVal);
+      String onOutOfMemoryVal = flags.getStringFlag("OnOutOfMemoryError");
+      String notifierScript = getScript("dd_oome_notifier");
+
+      if (onOutOfMemoryVal == null || onOutOfMemoryVal.isEmpty()) {
+        onOutOfMemoryVal = notifierScript;
+      } else if (!onOutOfMemoryVal.contains("dd_oome_notifier")) {
+        // we can chain scripts so let's preserve the original value in addition to our oome tracker
+        onOutOfMemoryVal = notifierScript + "; " + onOutOfMemoryVal;
+      }
+
+      // set the JVM flag
+      flags.setStringFlag("OnOutOfMemoryError", onOutOfMemoryVal);
+      if (LOG.isDebugEnabled()) {
+        String currentVal = flags.getStringFlag("OnOutOfMemoryError");
+        if (!currentVal.equals(onOutOfMemoryVal)) {
+          LOG.debug(
+              "Unable to set OnOutOfMemoryError flag to {}. OOME tracking may not work.",
+              currentVal);
+        }
+      }
+
+      OOMENotifierScriptInitializer.initialize(notifierScript);
     } catch (Throwable t) {
       logInitializationError(
           "Unexpected exception while initializing OOME notifier. OOMEs will not be tracked.", t);
     }
   }
 
+  private static String getScript(String scriptName) {
+    return System.getProperty("java.io.tmpdir") + "/" + getScriptFileName(scriptName) + " %p";
+  }
+
+  private static String getScriptFileName(String scriptName) {
+    return scriptName + "." + (Platform.isWindows() ? "bat" : "sh");
+  }
+
   private static void logInitializationError(String msg, Throwable t) {
     if (LOG.isDebugEnabled()) {
       LOG.warn("{}", msg, t);

dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/OOMENotifierScriptInitializer.java

@@ -1,12 +1,12 @@
 package com.datadog.crashtracking;
 
-import static com.datadog.crashtracking.ScriptInitializer.LOG;
-import static com.datadog.crashtracking.ScriptInitializer.PID_PREFIX;
-import static com.datadog.crashtracking.ScriptInitializer.RWXRWXRWX;
-import static com.datadog.crashtracking.ScriptInitializer.R_XR_XR_X;
-import static com.datadog.crashtracking.ScriptInitializer.findAgentJar;
-import static com.datadog.crashtracking.ScriptInitializer.getOomeNotifierTemplate;
-import static com.datadog.crashtracking.ScriptInitializer.writeConfig;
+import static com.datadog.crashtracking.Initializer.LOG;
+import static com.datadog.crashtracking.Initializer.PID_PREFIX;
+import static com.datadog.crashtracking.Initializer.RWXRWXRWX;
+import static com.datadog.crashtracking.Initializer.R_XR_XR_X;
+import static com.datadog.crashtracking.Initializer.findAgentJar;
+import static com.datadog.crashtracking.Initializer.getOomeNotifierTemplate;
+import static com.datadog.crashtracking.Initializer.writeConfig;
 import static java.nio.file.FileVisitResult.CONTINUE;
 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
 import static java.nio.file.attribute.PosixFilePermissions.asFileAttribute;

gradle/libs.versions.toml

@@ -29,7 +29,7 @@ moshi = '1.11.0'
 testcontainers = '1.20.1'
 jmc = "8.1.0"
 autoservice = "1.0-rc7"
-ddprof = "1.24.0"
+ddprof = "1.25.0"
 asm = "9.8"
 cafe_crypto = "0.1.0"
 lz4 = "1.7.1"