Increasing retry time at multiple places and force delete of consul namespace on cluster (#4596)

* Fixing flaky tests

* Setting eks version to 1.32

* Addition of force namespace deletion on cleanup of openshift

* Changing retry conditions for api_gateway_sink_test

* Fixing linting issue of return value not checked

Author: pajay-rao

acceptance/tests/api-gateway/api_gateway_kitchen_sink_test.go

@@ -133,7 +133,7 @@ func TestAPIGateway_KitchenSink(t *testing.T) {
 		httpRoute      gwv1beta1.HTTPRoute
 	)
 
-	counter = &retry.Counter{Count: 60, Wait: 2 * time.Second}
+	counter = &retry.Counter{Count: 60, Wait: 60 * time.Second}
 	retry.RunWith(counter, t, func(r *retry.R) {
 		var gateway gwv1beta1.Gateway
 		err = k8sClient.Get(context.Background(), types.NamespacedName{Name: "gateway", Namespace: "default"}, &gateway)
@@ -149,7 +149,7 @@ func TestAPIGateway_KitchenSink(t *testing.T) {
 		require.Len(r, gateway.Status.Listeners, 2)
 
 		// http route checks
-		counter = &retry.Counter{Count: 60, Wait: 10 * time.Second}
+		counter = &retry.Counter{Count: 60, Wait: 60 * time.Second}
 		retry.RunWith(counter, t, func(r *retry.R) {
 			err = k8sClient.Get(context.Background(), types.NamespacedName{Name: "http-route", Namespace: "default"}, &httpRoute)
 			require.NoError(r, err)

acceptance/tests/openshift/basic_openshift_test.go

@@ -33,16 +33,21 @@ func TestOpenshift_Basic(t *testing.T) {
 	logf.SetLogger(logr.New(nil))
 	logger.Log(t, "creating resources for OpenShift test")
 
-	cmd := exec.Command("kubectl", "apply", "-f", "../fixtures/cases/openshift/basic")
-	output, err := cmd.CombinedOutput()
+	counter := &retry.Counter{Count: 10, Wait: 120 * time.Second}
+	var err error
+	var output []byte
+	retry.RunWith(counter, t, func(r *retry.R) {
+		cmd := exec.Command("kubectl", "apply", "-f", "../fixtures/cases/openshift/basic")
+		output, err = cmd.CombinedOutput()
+		require.NoErrorf(r, err, "failed to create resources: %s", string(output))
+	})
+
 	helpers.Cleanup(t, cfg.NoCleanupOnFailure, cfg.NoCleanup, func() {
 		cmd := exec.Command("kubectl", "delete", "-f", "../fixtures/cases/openshift/basic")
 		output, err := cmd.CombinedOutput()
 		assert.NoErrorf(t, err, "failed to delete resources: %s", string(output))
 	})
 
-	require.NoErrorf(t, err, "failed to create resources: %s", string(output))
-
 	// Grab a kubernetes client so that we can verify binding
 	// behavior prior to issuing requests through the gateway.
 	ctx := suite.Environment().DefaultContext(t)
@@ -53,7 +58,7 @@ func TestOpenshift_Basic(t *testing.T) {
 	// On startup, the controller can take upwards of 1m to perform leader election,
 	// so we may need to wait a long time for the reconcile loop to run (hence the timeout).
 	var gatewayIP string
-	counter := &retry.Counter{Count: 120, Wait: 2 * time.Second}
+	counter = &retry.Counter{Count: 10, Wait: 120 * time.Second}
 	retry.RunWith(counter, t, func(r *retry.R) {
 		var gateway gwv1beta1.Gateway
 		err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "api-gateway", Namespace: "consul"}, &gateway)
@@ -73,7 +78,7 @@ func TestOpenshift_Basic(t *testing.T) {
 	}}
 
 	var resp *http.Response
-	counter = &retry.Counter{Count: 120, Wait: 2 * time.Second}
+	counter = &retry.Counter{Count: 10, Wait: 120 * time.Second}
 	retry.RunWith(counter, t, func(r *retry.R) {
 		resp, err = client.Get("https://" + gatewayIP)
 		require.NoErrorf(r, err, "request to API gateway failed: %s", err)

acceptance/tests/openshift/openshift_test_runner.go

@@ -1,17 +1,36 @@
 package openshift
 
 import (
+	"os/exec"
+	"strconv"
+	"testing"
+
 	"github.com/hashicorp/consul-k8s/acceptance/framework/config"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"os/exec"
-	"strconv"
-	"testing"
 )
 
 func newOpenshiftCluster(t *testing.T, cfg *config.TestConfig, secure, namespaceMirroring bool) {
-	cmd := exec.Command("helm", "repo", "add", "hashicorp", "https://helm.releases.hashicorp.com")
+	// Cleanup of old consul secret
+	cmd := exec.Command("kubectl", "delete", "secret", "-n", "consul", "consul-ent-license")
+	_, _ = cmd.CombinedOutput()
+
+	// Cleanup of old consul helm installtion and namespace
+	cmd = exec.Command("helm", "uninstall", "consul", "--namespace", "consul")
+	_, _ = cmd.CombinedOutput()
+
+	// Bypass finalizers for the consul namespace deletion.
+	ns := "consul"
+	cmd = exec.Command("bash", "-c", `kubectl get ns "`+ns+`" -o json | jq 'del(.spec.finalizers)' | kubectl replace --raw "/api/v1/namespaces/`+ns+`/finalize" -f -`)
+	_ = cmd.Run()
+
+	// Cleanup of old consul namespace
+	cmd = exec.Command("kubectl", "delete", "namespace", "consul")
+	_, _ = cmd.CombinedOutput()
+
+	// Add the hashicorp helm repo
+	cmd = exec.Command("helm", "repo", "add", "hashicorp", "https://helm.releases.hashicorp.com")
 	output, err := cmd.CombinedOutput()
 	require.NoErrorf(t, err, "failed to add hashicorp helm repo: %s", string(output))
 
@@ -59,6 +78,7 @@ func newOpenshiftCluster(t *testing.T, cfg *config.TestConfig, secure, namespace
 		"--set", "global.imageConsulDataplane="+cfg.ConsulDataplaneImage,
 		"--set", "global.enterpriseLicense.secretName=consul-ent-license",
 		"--set", "global.enterpriseLicense.secretKey=key",
+		"--set", "connectInject.apiGateway.manageExternalCRDs=true",
 	)
 
 	output, err = cmd.CombinedOutput()