Backport of fix: update mesh gateway rules for peering and partition handling into release/1.8.x (#5446)

backport of commit 90b0770

Co-authored-by: nitin-sachdev-29 <nitin.sachdev@hashicorp.com>

Author: hc-github-team-consul-core

.changelog/5423.txt

@@ -0,0 +1,4 @@
+```release-note:bug
+control-plane: fix mesh-gateway ACL policy generation for Admin Partition failover so Sameness Group clusters can form across local partitions.
+```
+

acceptance/tests/sameness/sameness_test.go

@@ -415,6 +415,20 @@ func TestFailover_Connect(t *testing.T) {
 					_, _, err := v.client.ConfigEntries().Set(intention, &api.WriteOptions{Partition: v.partition})
 					require.NoError(t, err)
 				}
+
+				// Validate that the generated mesh-gateway ACL policy grants the
+				// cross-partition service/node read permissions required for
+				// admin-partition Sameness Group failover targets. Without these
+				// permissions (nested under partition_prefix), a mesh gateway knows
+				// about a failover target in another local partition but cannot form
+				// the Envoy cluster for it, so failover silently fails.
+				// See https://github.com/hashicorp/consul-k8s/pull/5423.
+				for _, v := range testClusters {
+					if v.hasServer {
+						logger.Logf(t, "verifying mesh-gateway ACL policy on %s", v.name)
+						v.verifyMeshGatewayACLPolicy(t)
+					}
+				}
 			}
 
 			logger.Log(t, "creating exported services")
@@ -854,6 +868,59 @@ func (c *cluster) getCatalogService(t *testing.T, svc, ns, partition string) *ap
 	return resp[0]
 }
 
+// verifyMeshGatewayACLPolicy asserts that the mesh-gateway ACL policy generated by
+// server-acl-init grants cross-partition service and node read permissions nested
+// under a partition_prefix block. When admin partitions are enabled, these reads
+// are required so a mesh gateway can build the Envoy cluster for a Sameness Group
+// failover target that lives in another local partition. Prior to the fix in
+// https://github.com/hashicorp/consul-k8s/pull/5423 the reads were emitted outside
+// of partition_prefix (scoped to the gateway's own partition only), which caused
+// "Cluster not found" failures during admin-partition failover.
+func (c *cluster) verifyMeshGatewayACLPolicy(t *testing.T) {
+	policies, _, err := c.client.ACL().PolicyList(&api.QueryOptions{})
+	require.NoError(t, err)
+
+	var meshGWPolicyID string
+	for _, p := range policies {
+		if strings.HasPrefix(p.Name, "mesh-gateway-policy") {
+			meshGWPolicyID = p.ID
+			break
+		}
+	}
+	require.NotEmpty(t, meshGWPolicyID, "mesh-gateway-policy not found on cluster %s", c.name)
+
+	policy, _, err := c.client.ACL().PolicyRead(meshGWPolicyID, &api.QueryOptions{})
+	require.NoError(t, err)
+	rules := policy.Rules
+	logger.Logf(t, "mesh-gateway policy rules on %s:\n%s", c.name, rules)
+
+	// The partition_prefix block must exist and must contain the service/node
+	// read grants. We assert ordering (reads appear after partition_prefix and
+	// before the namespace "default" mesh-gateway write block) to ensure the
+	// reads are nested inside partition_prefix rather than scoped to the local
+	// partition only.
+	partitionIdx := strings.Index(rules, `partition_prefix "" {`)
+	require.NotEqual(t, -1, partitionIdx,
+		"mesh-gateway policy on %s is missing the partition_prefix block:\n%s", c.name, rules)
+
+	serviceIdx := strings.Index(rules, `service_prefix "" {`)
+	nodeIdx := strings.Index(rules, `node_prefix "" {`)
+	require.NotEqual(t, -1, serviceIdx, "mesh-gateway policy on %s is missing service_prefix read:\n%s", c.name, rules)
+	require.NotEqual(t, -1, nodeIdx, "mesh-gateway policy on %s is missing node_prefix read:\n%s", c.name, rules)
+
+	require.Greater(t, serviceIdx, partitionIdx,
+		"service_prefix read must be nested under partition_prefix in mesh-gateway policy on %s:\n%s", c.name, rules)
+	require.Greater(t, nodeIdx, partitionIdx,
+		"node_prefix read must be nested under partition_prefix in mesh-gateway policy on %s:\n%s", c.name, rules)
+
+	if nsWriteIdx := strings.Index(rules, `namespace "default" {`); nsWriteIdx != -1 {
+		require.Less(t, serviceIdx, nsWriteIdx,
+			"service_prefix read must be within the partition_prefix block (before the namespace \"default\" block) in mesh-gateway policy on %s:\n%s", c.name, rules)
+		require.Less(t, nodeIdx, nsWriteIdx,
+			"node_prefix read must be within the partition_prefix block (before the namespace \"default\" block) in mesh-gateway policy on %s:\n%s", c.name, rules)
+	}
+}
+
 type clusters map[string]*cluster
 
 func (c clusters) resetScale(t *testing.T) {

control-plane/subcommand/server-acl-init/rules.go

@@ -152,15 +152,38 @@ func (c *Command) meshGatewayRules() (string, error) {
 	// namespaces, it needs access to all namespaces. For peering, it requires the ability to list all peers which in
 	// enterprise requires peering:read on all partitions or in OSS requires a top level peering:read. Since we cannot
 	// determine whether we are using an enterprise or OSS consul image based on whether peering is enabled, we include
-	// both permissions here.
+	// both permissions here. When admin partitions are enabled, the default partition's mesh gateway additionally needs
+	// service and node read permissions under partition_prefix so it can form clusters for failover targets in other
+	// local partitions (e.g. Sameness Group failover). The partition_prefix block is only emitted for the default
+	// partition because partition rules are not valid in a policy scoped to a non-default admin partition; non-default
+	// partitions fall back to local namespace-scoped reads.
 	meshGatewayRulesTpl := `mesh = "write"
 {{- if .EnablePeering }}
 peering = "read"
+{{- end }}
 {{- if eq .PartitionName "default" }}
 partition_prefix "" {
+{{- if .EnablePeering }}
   peering = "read"
-}
 {{- end }}
+{{- if .EnableNamespaces }}
+  namespace_prefix "" {
+    node_prefix "" {
+      policy = "read"
+    }
+    service_prefix "" {
+      policy = "read"
+    }
+  }
+{{- else }}
+  node_prefix "" {
+    policy = "read"
+  }
+  service_prefix "" {
+    policy = "read"
+  }
+{{- end }}
+}
 {{- end }}
 {{- if .EnableNamespaces }}
 namespace "default" {
@@ -170,6 +193,9 @@ namespace "default" {
   }
 {{- if .EnableNamespaces }}
 }
+{{- end }}
+{{- if ne .PartitionName "default" }}
+{{- if .EnableNamespaces }}
 namespace_prefix "" {
 {{- end }}
   node_prefix "" {
@@ -181,6 +207,7 @@ namespace_prefix "" {
 {{- if .EnableNamespaces }}
 }
 {{- end }}
+{{- end }}
 `
 
 	return c.renderRules(meshGatewayRulesTpl)

control-plane/subcommand/server-acl-init/rules_test.go

@@ -204,7 +204,23 @@ peering = "read"
 peering = "read"
 partition_prefix "" {
   peering = "read"
+  node_prefix "" {
+    policy = "read"
+  }
+  service_prefix "" {
+    policy = "read"
+  }
 }
+  service "mesh-gateway" {
+     policy = "write"
+  }`,
+		},
+		{
+			Name:          "Peering is enabled with partition explicitly specified as non-default (ent non-default case)",
+			EnablePeering: true,
+			PartitionName: "non-default",
+			Expected: `mesh = "write"
+peering = "read"
   service "mesh-gateway" {
      policy = "write"
   }
@@ -216,27 +232,128 @@ partition_prefix "" {
   }`,
 		},
 		{
-			Name:          "Peering is enabled with partition explicitly specified as non-default (ent non-default case)",
-			EnablePeering: true,
-			PartitionName: "non-default",
+			Name:             "Peering and namespaces are enabled",
+			EnablePeering:    true,
+			EnableNamespaces: true,
 			Expected: `mesh = "write"
 peering = "read"
+namespace "default" {
   service "mesh-gateway" {
      policy = "write"
   }
+}
+namespace_prefix "" {
   node_prefix "" {
   	policy = "read"
   }
   service_prefix "" {
      policy = "read"
-  }`,
+  }
+}`,
 		},
 		{
-			Name:             "Peering and namespaces are enabled",
+			Name:             "Peering, namespaces, and default partition are enabled",
 			EnablePeering:    true,
 			EnableNamespaces: true,
+			PartitionName:    "default",
 			Expected: `mesh = "write"
 peering = "read"
+partition_prefix "" {
+  peering = "read"
+  namespace_prefix "" {
+    node_prefix "" {
+      policy = "read"
+    }
+    service_prefix "" {
+      policy = "read"
+    }
+  }
+}
+namespace "default" {
+  service "mesh-gateway" {
+     policy = "write"
+  }
+}`,
+		},
+		{
+			Name:             "Peering, namespaces, and non-default partition are enabled",
+			EnablePeering:    true,
+			EnableNamespaces: true,
+			PartitionName:    "non-default",
+			Expected: `mesh = "write"
+peering = "read"
+namespace "default" {
+  service "mesh-gateway" {
+     policy = "write"
+  }
+}
+namespace_prefix "" {
+  node_prefix "" {
+  	policy = "read"
+  }
+  service_prefix "" {
+     policy = "read"
+  }
+}`,
+		},
+		{
+			// Sameness Group failover across local admin partitions only
+			// (no cluster peering). The partition_prefix read grants must
+			// still be present on the default partition's gateway so it can
+			// form failover clusters for services in sibling partitions.
+			Name:             "Namespaces and default partition are enabled, peering disabled",
+			EnablePeering:    false,
+			EnableNamespaces: true,
+			PartitionName:    "default",
+			Expected: `mesh = "write"
+partition_prefix "" {
+  namespace_prefix "" {
+    node_prefix "" {
+      policy = "read"
+    }
+    service_prefix "" {
+      policy = "read"
+    }
+  }
+}
+namespace "default" {
+  service "mesh-gateway" {
+     policy = "write"
+  }
+}`,
+		},
+		{
+			// Partitions enabled with neither peering nor namespaces. The
+			// chart forbids partitions without namespaces, but the template
+			// must still render valid HCL via the else branch.
+			Name:             "Default partition is enabled, peering and namespaces disabled",
+			EnablePeering:    false,
+			EnableNamespaces: false,
+			PartitionName:    "default",
+			Expected: `mesh = "write"
+partition_prefix "" {
+  node_prefix "" {
+    policy = "read"
+  }
+  service_prefix "" {
+    policy = "read"
+  }
+}
+  service "mesh-gateway" {
+     policy = "write"
+  }`,
+		},
+		{
+			// Regression guard: a non-default admin partition must NOT emit a
+			// partition_prefix block, because partition rules are invalid in a
+			// policy scoped to a non-default partition. server-acl-init would
+			// otherwise fail to create the mesh-gateway policy, leaving the
+			// gateway pod unready (see PR #5423 follow-up).
+			Name:             "Namespaces and non-default partition are enabled, peering disabled",
+			EnablePeering:    false,
+			EnableNamespaces: true,
+			PartitionName:    "non-default",
+			Expected: `mesh = "write"
 namespace "default" {
   service "mesh-gateway" {
      policy = "write"
@@ -251,6 +368,24 @@ namespace_prefix "" {
   }
 }`,
 		},
+		{
+			// Regression guard: non-default partition without namespaces also
+			// must not emit a partition_prefix block.
+			Name:             "Non-default partition is enabled, peering and namespaces disabled",
+			EnablePeering:    false,
+			EnableNamespaces: false,
+			PartitionName:    "non-default",
+			Expected: `mesh = "write"
+  service "mesh-gateway" {
+     policy = "write"
+  }
+  node_prefix "" {
+  	policy = "read"
+  }
+  service_prefix "" {
+     policy = "read"
+  }`,
+		},
 	}
 
 	for _, tt := range cases {