@@ -5,15 +5,149 @@ import (
55 "github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
66 "github.com/stretchr/testify/assert"
77 "github.com/stretchr/testify/require"
8+ "os"
89 "os/exec"
910 "strconv"
11+ "strings"
12+ "syscall"
1013 "testing"
1114)
1215
16+ func interruptProcess (t * testing.T ) {
17+ p , err := os .FindProcess (os .Getpid ())
18+ if err != nil {
19+ t .Logf ("Failed to find process: %v" , err )
20+ return
21+ }
22+ err = p .Signal (syscall .SIGINT )
23+ if err != nil {
24+ t .Logf ("Failed to send interrupt signal: %v" , err )
25+ }
26+ }
27+ func removeNamespaceFinalizer (t * testing.T , namespace string ) {
28+ cmd := exec .Command ("kubectl" , "patch" , "namespace" , namespace ,
29+ "--type=json" , "-p" , `[{"op": "remove", "path": "/spec/finalizers"}]` )
30+
31+ if output , err := cmd .CombinedOutput (); err != nil {
32+ t .Logf ("Error removing namespace finalizer: %v\n Output: %s\n " , err , output )
33+ os .Exit (1 )
34+ }
35+ verifyNamespaceDeletion (t , namespace )
36+ }
37+
38+ func verifyNamespaceDeletion (t * testing.T , namespace string ) {
39+ checkCmd := exec .Command ("kubectl" , "get" , "ns" , namespace )
40+ if checkErr := checkCmd .Run (); checkErr != nil {
41+ t .Logf ("Namespace %s deleted successfully\n " , namespace )
42+ return
43+ }
44+
45+ t .Logf ("Namespace still exists. Additional cleanup might be required" )
46+ interruptProcess (t )
47+ }
48+
49+ func gatewayControllerDiagnostic (t * testing.T , namespace string ) {
50+ // Add diagnostic logging for pods for controller identification
51+ lsNamespaceCmd := exec .Command ("oc" , "get" , "namespaces" , "-o" , "wide" )
52+ lsNamespaceOutput , _ := lsNamespaceCmd .CombinedOutput ()
53+ t .Logf ("Namespaces in cluster:\n %s" , string (lsNamespaceOutput ))
54+
55+ podInfoCmd := exec .Command ("kubectl" , "get" , "pods" , "-n" , namespace )
56+ podInfoOutput , _ := podInfoCmd .CombinedOutput ()
57+ t .Logf ("Pod status in namespace %s before cleanup:\n %s" , namespace , string (podInfoOutput ))
58+
59+ podInfoCmd = exec .Command ("kubectl" , "get" , "pods" , "-n" , "kube-system" )
60+ podInfoOutput , _ = podInfoCmd .CombinedOutput ()
61+ t .Logf ("Pod status in namespace %s before cleanup:\n %s" , namespace , string (podInfoOutput ))
62+ }
63+
64+ func checkAndDeleteNamespace (t * testing.T ) {
65+ // There are some commands in this function
66+ //which are not being replaced with this variables and need manual replacement of namespace
67+ namespace := "consul"
68+ // Check if namespace exists and its status
69+ nsCheckCmd := exec .Command ("kubectl" , "get" , "namespace" , namespace , "-o" , "json" )
70+ nsOutput , _ := nsCheckCmd .CombinedOutput ()
71+ t .Logf ("Consul namespace status before cleanup:\n %s" , string (nsOutput ))
72+
73+ // Add diagnostic logging before attempting cleanup
74+ logCmd := exec .Command ("kubectl" , "get" , "all" , "-n" , namespace )
75+ logOutput , _ := logCmd .CombinedOutput ()
76+ t .Logf ("Resources in consul namespace before cleanup:\n %s" , string (logOutput ))
77+
78+ // find gateway controller information and logs
79+ gatewayControllerDiagnostic (t , namespace )
80+ // Force cleanup of any stuck resources in the namespace (if it still exists)
81+ t .Log ("Checking for any stuck resources..." )
82+ forceCleanupCmd := exec .Command ("bash" , "-c" , `
83+ # Try to find finalizers on the namespace
84+ FINALIZERS=$(kubectl get namespace consul -o json 2>/dev/null | jq '.spec.finalizers' 2>/dev/null)
85+ if [ ! -z "$FINALIZERS" ] && [ "$FINALIZERS" != "null" ]; then
86+ echo "Found finalizers on namespace consul"
87+ echo $FINALIZERS
88+ # Remove finalizers from namespace to force deletion
89+ # kubectl get namespace consul -o json | jq '.spec.finalizers = []' | kubectl replace --raw "/api/v1/namespaces/consul/finalize" -f -
90+ fi
91+ if kubectl get namespace consul > /dev/null 2>&1; then
92+ # Check for gateway resources
93+ GATEWAYS=$(kubectl get gateways.gateway.networking.k8s.io -n consul -o json 2>/dev/null || echo "")
94+ echo $GATEWAYS
95+ fi
96+ ` )
97+ forceOutput , _ := forceCleanupCmd .CombinedOutput ()
98+ t .Logf ("Force cleanup result:\n %s" , string (forceOutput ))
99+
100+ // Get remaining Gateways
101+ getCmd := exec .Command ("kubectl" , "get" , "gateways.gateway.networking.k8s.io" , "-n" , namespace ,
102+ "-o=jsonpath={.items[*].metadata.name}" )
103+ output , err := getCmd .CombinedOutput ()
104+ t .Logf ("Gateway resource check result:\n %s" , string (output ))
105+
106+ if err != nil {
107+ t .Logf ("Error getting gateways: %v\n " , err )
108+ return
109+ }
110+ cleanedOutput := strings .TrimSpace (string (output ))
111+ if cleanedOutput == "" {
112+ t .Logf ("No gateways found, removing namespace finalizer" )
113+ removeNamespaceFinalizer (t , namespace )
114+ return
115+ }
116+ if len (cleanedOutput ) > 0 {
117+ // Remove finalizers from each gateway
118+ patchCmd := exec .Command ("kubectl" , "patch" , string (output ), "-n" , namespace ,
119+ "--type=json" , "-p" , `[{"op": "remove", "path": "/metadata/finalizers"}]` )
120+ patchOutput , patchErr := patchCmd .CombinedOutput ()
121+ if patchErr != nil {
122+ t .Logf ("Error patching gateway: %v\n Output: %s\n " , patchErr , patchOutput )
123+ return
124+ }
125+ t .Logf ("Finalizers removed successfully" )
126+ removeNamespaceFinalizer (t , namespace )
127+ }
128+ t .Log ("Attempting to delete consul namespace if it exists..." )
129+ cleanupCmd := exec .Command ("kubectl" , "delete" , "namespace" , "consul" , "--ignore-not-found=true" )
130+ cleanupOutput , cleanupErr := cleanupCmd .CombinedOutput ()
131+ // We don't check error here since it's just precautionary cleanup
132+ t .Logf ("Namespace deletion attempt result: %v\n Output: %s" , cleanupErr , string (cleanupOutput ))
133+
134+ // Wait for namespace to be fully deleted before proceeding
135+ t .Log ("Waiting for consul namespace to be fully deleted..." )
136+ waitCmd := exec .Command ("kubectl" , "wait" , "--for=delete" , "namespace/consul" , "--timeout=30s" )
137+ waitOutput , waitErr := waitCmd .CombinedOutput () // Ignore errors, as this will error if the namespace doesn't exist at all
138+ t .Logf ("Wait result: %v\n Output: %s" , waitErr , string (waitOutput ))
139+
140+ // Verify namespace deletion
141+ verifyNamespaceDeletion (t , namespace )
142+ }
143+
13144func newOpenshiftCluster (t * testing.T , cfg * config.TestConfig , secure , namespaceMirroring bool ) {
14145 cmd := exec .Command ("helm" , "repo" , "add" , "hashicorp" , "https://helm.releases.hashicorp.com" )
15146 output , err := cmd .CombinedOutput ()
16- require .NoErrorf (t , err , "failed to add hashicorp helm repo: %s" , string (output ))
147+ require .NoErrorf (t , err , "failed to add hashicorp helm repo : %s" , string (output ))
148+
149+ // Check for any stuck resources in the namespace and force cleanup if necessary
150+ checkAndDeleteNamespace (t )
17151
18152 // FUTURE for some reason NewHelmCluster creates a consul server pod that runs as root which
19153 // isn't allowed in OpenShift. In order to test OpenShift properly, we have to call helm and k8s
@@ -26,7 +160,7 @@ func newOpenshiftCluster(t *testing.T, cfg *config.TestConfig, secure, namespace
26160 assert .NoErrorf (t , err , "failed to delete namespace: %s" , string (output ))
27161 })
28162
29- require .NoErrorf (t , err , "failed to add hashicorp helm repo : %s" , string (output ))
163+ require .NoErrorf (t , err , "failed to create namespace : %s" , string (output ))
30164
31165 cmd = exec .Command ("kubectl" , "create" , "secret" , "generic" ,
32166 "consul-ent-license" ,
0 commit comments