WIP: no-jira: azure: Fix the dualstack errors

pkg/infrastructure/azure/azure.go

@@ -448,9 +448,14 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 	var extLBFQDN string
 	if in.InstallConfig.Config.PublicAPI() {
 		var publicIPv6 *armnetwork.PublicIPAddress
+		v4InfraID := in.InfraID
+		v6InfraID := ""
+		if in.InstallConfig.Config.Azure.IPFamily.DualStackEnabled() {
+			v6InfraID = fmt.Sprintf("%s-v6", in.InfraID)
+		}
 		publicIP, err := createPublicIP(ctx, &pipInput{
 			name:          fmt.Sprintf("%s-pip-v4", in.InfraID),
-			infraID:       in.InfraID,
+			infraID:       v4InfraID,
 			region:        in.InstallConfig.Config.Azure.Region,
 			resourceGroup: resourceGroupName,
 			pipClient:     networkClientFactory.NewPublicIPAddressesClient(),
@@ -464,7 +469,7 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 		if in.InstallConfig.Config.Azure.IPFamily.DualStackEnabled() {
 			publicIPv6, err = createPublicIP(ctx, &pipInput{
 				name:          fmt.Sprintf("%s-pip-v6", in.InfraID),
-				infraID:       in.InfraID,
+				infraID:       v6InfraID,
 				region:        in.InstallConfig.Config.Azure.Region,
 				resourceGroup: resourceGroupName,
 				pipClient:     networkClientFactory.NewPublicIPAddressesClient(),
@@ -585,6 +590,36 @@ func (p *Provider) PostProvision(ctx context.Context, in clusterapi.PostProvisio
 			return fmt.Errorf("failed to associate control plane VMs with external load balancer: %w", err)
 		}
 
+		if in.InstallConfig.Config.Azure.IPFamily.DualStackEnabled() {
+			bootstrapNicName := fmt.Sprintf("%s-bootstrap-nic", in.InfraID)
+			nicClient := p.NetworkClientFactory.NewInterfacesClient()
+			bootstrapNic, err := nicClient.Get(ctx, p.ResourceGroupName, bootstrapNicName, nil)
+			if err != nil {
+				return fmt.Errorf("failed to get bootstrap nic: %w", err)
+			}
+			for _, ipconfig := range bootstrapNic.Properties.IPConfigurations {
+				if ipconfig.Properties.PrivateIPAddressVersion != nil && *ipconfig.Properties.PrivateIPAddressVersion == armnetwork.IPVersionIPv6 {
+					for _, pool := range p.lbBackendAddressPools {
+						if pool.Name != nil && strings.HasSuffix(*pool.Name, "-v6") {
+							ipconfig.Properties.LoadBalancerBackendAddressPools = append(
+								ipconfig.Properties.LoadBalancerBackendAddressPools,
+								pool,
+							)
+						}
+					}
+				}
+			}
+			pollerResp, err := nicClient.BeginCreateOrUpdate(ctx, p.ResourceGroupName, bootstrapNicName, bootstrapNic.Interface, nil)
+			if err != nil {
+				return fmt.Errorf("failed to update bootstrap nic with IPv6 backend pools: %w", err)
+			}
+			_, err = pollerResp.PollUntilDone(ctx, nil)
+			if err != nil {
+				return fmt.Errorf("failed to update bootstrap nic with IPv6 backend pools: %w", err)
+			}
+			logrus.Debugf("associated bootstrap NIC with IPv6 backend pools")
+		}
+
 		sshRuleName := fmt.Sprintf("%s_ssh_in", in.InfraID)
 
 		loadBalancerName := in.InfraID
@@ -630,27 +665,6 @@ func (p *Provider) PostProvision(ctx context.Context, in clusterapi.PostProvisio
 
 		// For dual-stack, create IPv6 inbound rule for SSH access to bootstrap.
 		if in.InstallConfig.Config.Azure.IPFamily.DualStackEnabled() {
-			publicIPv6outbound, err := createPublicIP(ctx, &pipInput{
-				name:          fmt.Sprintf("%s-pip-v6-outbound-lb", in.InfraID),
-				infraID:       in.InfraID,
-				region:        in.InstallConfig.Config.Azure.Region,
-				resourceGroup: p.ResourceGroupName,
-				pipClient:     p.NetworkClientFactory.NewPublicIPAddressesClient(),
-				tags:          p.Tags,
-				ipversion:     armnetwork.IPVersionIPv6,
-			})
-			if err != nil {
-				return fmt.Errorf("failed to create public ipv6 for outbound ipv6 lb: %w", err)
-			}
-			logrus.Debugf("created public ipv6 for outbound ipv6 lb: %s", *publicIPv6outbound.ID)
-
-			// Update the outbound node IPv6 load balancer.
-			outboundLBName := fmt.Sprintf("%s-ipv6-outbound-node-lb", in.InfraID)
-			err = updateOutboundIPv6LoadBalancer(ctx, publicIPv6outbound, p.NetworkClientFactory.NewLoadBalancersClient(), p.ResourceGroupName, outboundLBName, in.InfraID)
-			if err != nil {
-				return fmt.Errorf("failed to set public ipv6 to outbound ipv6 lb: %w", err)
-			}
-			logrus.Debugf("updated outbound ipv6 lb %s with public ipv6: %s", outboundLBName, *publicIPv6outbound.ID)
 			frontendIPv6ConfigName := "public-lb-ip-v6"
 			sshRuleNameV6 := fmt.Sprintf("%s_ssh_in_v6", in.InfraID)
 			frontendIPv6ConfigID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers/%s/frontendIPConfigurations/%s",

pkg/infrastructure/azure/network.go

@@ -73,6 +73,15 @@ type inboundNatRuleInput struct {
 }
 
 func createPublicIP(ctx context.Context, in *pipInput) (*armnetwork.PublicIPAddress, error) {
+	properties := &armnetwork.PublicIPAddressPropertiesFormat{
+		PublicIPAddressVersion:   to.Ptr(in.ipversion),
+		PublicIPAllocationMethod: to.Ptr(armnetwork.IPAllocationMethodStatic),
+	}
+	if in.infraID != "" {
+		properties.DNSSettings = &armnetwork.PublicIPAddressDNSSettings{
+			DomainNameLabel: to.Ptr(in.infraID),
+		}
+	}
 	pollerResp, err := in.pipClient.BeginCreateOrUpdate(
 		ctx,
 		in.resourceGroup,
@@ -84,14 +93,8 @@ func createPublicIP(ctx context.Context, in *pipInput) (*armnetwork.PublicIPAddr
 				Name: to.Ptr(armnetwork.PublicIPAddressSKUNameStandard),
 				Tier: to.Ptr(armnetwork.PublicIPAddressSKUTierRegional),
 			},
-			Properties: &armnetwork.PublicIPAddressPropertiesFormat{
-				PublicIPAddressVersion:   to.Ptr(in.ipversion),
-				PublicIPAllocationMethod: to.Ptr(armnetwork.IPAllocationMethodStatic),
-				DNSSettings: &armnetwork.PublicIPAddressDNSSettings{
-					DomainNameLabel: to.Ptr(in.infraID),
-				},
-			},
-			Tags: in.tags,
+			Properties: properties,
+			Tags:       in.tags,
 		},
 		nil,
 	)
@@ -764,31 +767,3 @@ func associateNatGatewayToSubnet(ctx context.Context, in natGatewayInput) error
 	}
 	return nil
 }
-
-func updateOutboundIPv6LoadBalancer(ctx context.Context, pipv6 *armnetwork.PublicIPAddress, lbClient *armnetwork.LoadBalancersClient, resourceGroup, loadBalancerName, infraID string) error {
-	outboundIPv6LB, err := lbClient.Get(ctx, resourceGroup, loadBalancerName, nil)
-	if err != nil {
-		return fmt.Errorf("failed to get external load balancer: %w", err)
-	}
-
-	loadBalancer := outboundIPv6LB.LoadBalancer
-	loadBalancer.Properties.FrontendIPConfigurations = append(loadBalancer.Properties.FrontendIPConfigurations, &armnetwork.FrontendIPConfiguration{
-		Name: to.Ptr(fmt.Sprintf("%s-frontend-ipv6", infraID)),
-		Properties: &armnetwork.FrontendIPConfigurationPropertiesFormat{
-			PrivateIPAllocationMethod: to.Ptr(armnetwork.IPAllocationMethodDynamic),
-			PublicIPAddress:           pipv6,
-		},
-	})
-
-	pollerResp, err := lbClient.BeginCreateOrUpdate(ctx,
-		resourceGroup,
-		loadBalancerName,
-		loadBalancer, nil)
-
-	if err != nil {
-		return fmt.Errorf("cannot update outbound node ipv6 load balancer: %w", err)
-	}
-
-	_, err = pollerResp.PollUntilDone(ctx, nil)
-	return err
-}

pkg/types/validation/installconfig.go

@@ -381,6 +381,9 @@ func validateNetworkingIPVersion(n *types.Networking, p *types.Platform) field.E
 		switch {
 		case p.Azure != nil && experimentalDualStackEnabled:
 			logrus.Warnf("Using experimental Azure dual-stack support")
+			if p.Azure.IPFamily == network.DualStackIPv6Primary {
+				allowV6Primary = true
+			}
 		case p.BareMetal != nil:
 			// We now support ipv6-primary dual stack on baremetal
 			allowV6Primary = true
@@ -424,7 +427,7 @@ func validateNetworkingIPVersion(n *types.Networking, p *types.Platform) field.E
 				allErrs = append(allErrs, field.Invalid(field.NewPath("networking", k), strings.Join(ipnetworksToStrings(addresses[k]), ", "), "dual-stack IPv4/IPv6 requires an IPv4 network in this list"))
 			}
 
-			allErrs = append(allErrs, validateNetworkEntryOrder(p, v, addresses[k], allowV6Primary, field.NewPath("networking", k))...)
+			allErrs = append(allErrs, validateNetworkEntryOrder(p, v, addresses[k], allowV6Primary, k, field.NewPath("networking", k))...)
 		}
 
 	case hasIPv6:
@@ -475,7 +478,7 @@ func validateNetworkingIPVersion(n *types.Networking, p *types.Platform) field.E
 // - IPv4 primary dual-stack: IPv4 CIDR first in list
 // - IPv6 primary dual-stack: IPv6 CIDR first in list
 // Some platforms have an explicit field to define the dual-stack variant, for example, platform.aws.ipFamily on AWS.
-func validateNetworkEntryOrder(p *types.Platform, ipAddressType ipAddressType, networks []ipnet.IPNet, allowV6Primary bool, fldPath *field.Path) field.ErrorList {
+func validateNetworkEntryOrder(p *types.Platform, ipAddressType ipAddressType, networks []ipnet.IPNet, allowV6Primary bool, networkType string, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 
 	// If missing either IPv4 or IPv6 CIDR, order validation is not applicable
@@ -495,6 +498,23 @@ func validateNetworkEntryOrder(p *types.Platform, ipAddressType ipAddressType, n
 		if ipFamily == network.DualStackIPv6Primary && ipAddressType.Primary == corev1.IPv4Protocol {
 			allErrs = append(allErrs, field.Invalid(fldPath, strings.Join(ipnetworksToStrings(networks), ", "), "DualStackIPv6Primary requires an IPv6 network first in this list"))
 		}
+	case p.Azure != nil:
+		ipFamily := p.Azure.IPFamily
+
+		// Azure nodes always have IPv4 as the primary NIC address, so serviceNetwork
+		// must have IPv4 first regardless of ipFamily. The kube-apiserver requires the
+		// primary service IP family to match the node's address family.
+		if networkType == networkTypeService && ipAddressType.Primary != corev1.IPv4Protocol {
+			allErrs = append(allErrs, field.Invalid(fldPath, strings.Join(ipnetworksToStrings(networks), ", "), "Azure requires an IPv4 service network first in this list because node primary addresses are always IPv4"))
+		}
+
+		if ipFamily == network.DualStackIPv4Primary && ipAddressType.Primary == corev1.IPv6Protocol && networkType != networkTypeService {
+			allErrs = append(allErrs, field.Invalid(fldPath, strings.Join(ipnetworksToStrings(networks), ", "), "DualStackIPv4Primary requires an IPv4 network first in this list"))
+		}
+
+		if ipFamily == network.DualStackIPv6Primary && ipAddressType.Primary == corev1.IPv4Protocol && networkType != networkTypeService {
+			allErrs = append(allErrs, field.Invalid(fldPath, strings.Join(ipnetworksToStrings(networks), ", "), "DualStackIPv6Primary requires an IPv6 network first in this list"))
+		}
 	default:
 		// For platforms that don't support IPv6-primary dual-stack, reject configurations with IPv6 CIDRs listed first.
 		if !allowV6Primary && ipAddressType.Primary != corev1.IPv4Protocol {

pkg/types/validation/installconfig_test.go

@@ -3,6 +3,7 @@ package validation
 import (
 	"fmt"
 	"net"
+	"os"
 	"testing"
 
 	"github.com/aws/smithy-go/ptr"
@@ -1976,6 +1977,84 @@ func TestValidateInstallConfig(t *testing.T) {
 			}(),
 			expectedError: `networking.serviceNetwork: Invalid value: "ffd1::/112": when installing dual-stack IPv4/IPv6 you must provide two service networks, one for each IP address type`,
 		},
+		{
+			name: "azure: valid dual-stack with DualStackIPv6Primary and IPv4-first serviceNetwork",
+			installConfig: func() *types.InstallConfig {
+				c := validInstallConfig()
+				c.Platform = types.Platform{Azure: validAzurePlatform()}
+				c.Platform.Azure.IPFamily = network.DualStackIPv6Primary
+				c.FeatureSet = configv1.CustomNoUpgrade
+				c.FeatureGates = []string{"AzureDualStackInstall=true"}
+				c.Networking = validPrimaryV6DualStackNetworkingConfig()
+				c.Networking.ServiceNetwork = []ipnet.IPNet{
+					*ipnet.MustParseCIDR("172.30.0.0/16"),
+					*ipnet.MustParseCIDR("ffd1::/112"),
+				}
+				return c
+			}(),
+			restoreFnFactory: func(_ *types.InstallConfig) func() {
+				os.Setenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK", "true")
+				return func() {
+					os.Unsetenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK")
+				}
+			},
+		},
+		{
+			name: "azure: valid dual-stack with DualStackIPv4Primary",
+			installConfig: func() *types.InstallConfig {
+				c := validInstallConfig()
+				c.Platform = types.Platform{Azure: validAzurePlatform()}
+				c.Platform.Azure.IPFamily = network.DualStackIPv4Primary
+				c.FeatureSet = configv1.CustomNoUpgrade
+				c.FeatureGates = []string{"AzureDualStackInstall=true"}
+				c.Networking = validDualStackNetworkingConfig()
+				return c
+			}(),
+			restoreFnFactory: func(_ *types.InstallConfig) func() {
+				os.Setenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK", "true")
+				return func() {
+					os.Unsetenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK")
+				}
+			},
+		},
+		{
+			name: "azure: invalid dual-stack with IPv6-first serviceNetwork",
+			installConfig: func() *types.InstallConfig {
+				c := validInstallConfig()
+				c.Platform = types.Platform{Azure: validAzurePlatform()}
+				c.Platform.Azure.IPFamily = network.DualStackIPv6Primary
+				c.FeatureSet = configv1.CustomNoUpgrade
+				c.FeatureGates = []string{"AzureDualStackInstall=true"}
+				c.Networking = validPrimaryV6DualStackNetworkingConfig()
+				return c
+			}(),
+			expectedError: `networking.serviceNetwork: Invalid value: "ffd1::/112, 172.30.0.0/16": Azure requires an IPv4 service network first in this list because node primary addresses are always IPv4`,
+			restoreFnFactory: func(_ *types.InstallConfig) func() {
+				os.Setenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK", "true")
+				return func() {
+					os.Unsetenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK")
+				}
+			},
+		},
+		{
+			name: "azure: invalid dual-stack with DualStackIPv4Primary but IPv6-primary networks",
+			installConfig: func() *types.InstallConfig {
+				c := validInstallConfig()
+				c.Platform = types.Platform{Azure: validAzurePlatform()}
+				c.Platform.Azure.IPFamily = network.DualStackIPv4Primary
+				c.FeatureSet = configv1.CustomNoUpgrade
+				c.FeatureGates = []string{"AzureDualStackInstall=true"}
+				c.Networking = validPrimaryV6DualStackNetworkingConfig()
+				return c
+			}(),
+			expectedError: `^\Q[networking.clusterNetwork: Invalid value: "ffd2::/48, 192.168.1.0/24": DualStackIPv4Primary requires an IPv4 network first in this list, networking.machineNetwork: Invalid value: "ffd0::/48, 10.0.0.0/16": DualStackIPv4Primary requires an IPv4 network first in this list, networking.serviceNetwork: Invalid value: "ffd1::/112, 172.30.0.0/16": Azure requires an IPv4 service network first in this list because node primary addresses are always IPv4]\E$`,
+			restoreFnFactory: func(_ *types.InstallConfig) func() {
+				os.Setenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK", "true")
+				return func() {
+					os.Unsetenv("OPENSHIFT_INSTALL_EXPERIMENTAL_DUAL_STACK")
+				}
+			},
+		},
 		{
 			name: "invalid IPv6 hostprefix",
 			installConfig: func() *types.InstallConfig {