[8.18] Unmute docker tests, add more logging and increase startup timeout (#131203) (#131295)

* Unmute docker tests, add more logging and increase startup timeout (#131203)

Unmuting 33 DockerTests with added additional logging on failures.
Also, bumping docker startup timeout to account for slow security
auto-configuration, and additionally moving CA certificates inside
the retry while loop.

Resolves #129808
Resolves #123437
Resolves #120911
Resolves #120914
Resolves #120910
Resolves #120918
Resolves #120925
Resolves #124990
Resolves #116636
Resolves #125680
Resolves #116147
Resolves #126232
Resolves #126145
Resolves #116866
Resolves #126936
Resolves #127081
Resolves #127158
Resolves #127138
Resolves #128018
Resolves #128031
Resolves #128110
Resolves #128006
Resolves #128174
Resolves #128144
Resolves #128165
Resolves #128075
Resolves #128117
Resolves #128121
Resolves #128115
Resolves #128230
Resolves #128120
Resolves #128996
Resolves #128867

(cherry picked from commit 4806156)

# Conflicts:
#	muted-tests.yml

* call dumpDebug

Author: slobodanadamovic

muted-tests.yml

@@ -369,8 +369,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=snapshot/20_operator_privileges_disabled/Operator only settings can be set and restored by non-operator user when operator privileges is disabled}
   issue: https://github.com/elastic/elasticsearch/issues/120973
-- class: org.elasticsearch.packaging.test.DockerTests
-  issue: https://github.com/elastic/elasticsearch/issues/120978
 - class: org.elasticsearch.xpack.security.authc.service.ServiceAccountIT
   method: testAuthenticateShouldNotFallThroughInCaseOfFailure
   issue: https://github.com/elastic/elasticsearch/issues/120902
@@ -484,4 +482,4 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/115528
 - class: org.elasticsearch.packaging.test.EnrollmentProcessTests
   method: test20DockerAutoFormCluster
-  issue: https://github.com/elastic/elasticsearch/issues/128113
+  issue: https://github.com/elastic/elasticsearch/issues/128113

qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java

@@ -145,6 +145,10 @@ public abstract class PackagingTestCase extends Assert {
         @Override
         protected void failed(Throwable e, Description description) {
             failed = true;
+            if (installation != null && installation.distribution.isDocker()) {
+                logger.warn("Test {} failed. Printing logs for failed test...", description.getMethodName());
+                dumpDebug();
+            }
         }
     };
 

qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -150,7 +150,19 @@ private static HttpResponse execute(Request request, String username, String pas
             executor.auth(username, password);
             executor.authPreemptive(new HttpHost("localhost", 9200));
         }
-        return executor.execute(request).returnResponse();
+        try {
+            return executor.execute(request).returnResponse();
+        } catch (Exception e) {
+            logger.warn(
+                "Failed to execute request [{}] with username/password [{}/{}] and caCert [{}]",
+                request.toString(),
+                username,
+                password,
+                caCert,
+                e
+            );
+            throw e;
+        }
     }
 
     // polls every two seconds for Elasticsearch to be running on 9200
@@ -238,14 +250,13 @@ public static void waitForElasticsearch(
         long timeElapsed = 0;
         boolean started = false;
         Throwable thrownException = null;
-        if (caCert == null) {
-            caCert = getCaCert(installation);
-        }
 
         while (started == false && timeElapsed < waitTime) {
             if (System.currentTimeMillis() - lastRequest > requestInterval) {
+                if (caCert == null) {
+                    caCert = getCaCert(installation);
+                }
                 try {
-
                     final HttpResponse response = execute(
                         Request.Get((caCert != null ? "https" : "http") + "://localhost:9200/_cluster/health")
                             .connectTimeout((int) timeoutLength)
@@ -276,7 +287,7 @@ public static void waitForElasticsearch(
                     }
                     started = true;
 
-                } catch (IOException e) {
+                } catch (Exception e) {
                     if (thrownException == null) {
                         thrownException = e;
                     } else {

qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java

@@ -73,7 +73,7 @@ public class Docker {
     public static final Shell sh = new Shell();
     public static final DockerShell dockerShell = new DockerShell();
     public static final int STARTUP_SLEEP_INTERVAL_MILLISECONDS = 1000;
-    public static final int STARTUP_ATTEMPTS_MAX = 30;
+    public static final int STARTUP_ATTEMPTS_MAX = 45;
 
     private static final String ELASTICSEARCH_FULL_CLASSNAME = "org.elasticsearch.bootstrap.Elasticsearch";
     private static final String FIND_ELASTICSEARCH_PROCESS = "for pid in $(ps -eo pid,comm | grep java | awk '\\''{print $1}'\\''); "