NatGateway integration – step 2

Allow users to bring their own public ip addresses and/or public ip ranges/prefixes which should be attached to the NatGateway.

charts/internal/azure-infra/templates/main.tf

@@ -77,7 +77,20 @@ resource "azurerm_network_security_group" "workers" {
 #===============================================
 #= NAT Gateway
 #===============================================
-
+{{ if .Values.natGateway -}}
+{{ range $index, $ip := .Values.natGateway.ipAddresses }}
+data "azurerm_public_ip" "nat-ip-{{ $index }}" {
+  name                = "{{ $ip.name }}"
+  resource_group_name = "{{ $ip.resourceGroup }}"
+}
+{{- end }}
+{{ range $index, $ip := .Values.natGateway.ipAddressRanges }}
+data "azurerm_public_ip_prefix" "nat-iprange-prefix-{{ $index }}" {
+  name                = "{{ $ip.name }}"
+  resource_group_name = "{{ $ip.resourceGroup }}"
+}
+{{- end }}
+{{ else -}}
 resource "azurerm_public_ip" "natip" {
   name                = "{{ required "clusterName is required" .Values.clusterName }}-nat-ip"
   location            = "{{ required "azure.region is required" .Values.azure.region }}"
@@ -89,6 +102,7 @@ resource "azurerm_public_ip" "natip" {
   allocation_method   = "Static"
   sku                 = "Standard"
 }
+{{- end }}
 
 resource "azurerm_nat_gateway" "nat" {
   name                    = "{{ required "clusterName is required" .Values.clusterName }}-nat-gateway"
@@ -99,7 +113,12 @@ resource "azurerm_nat_gateway" "nat" {
   resource_group_name     = "${data.azurerm_resource_group.rg.name}"
   {{- end }}
   sku_name                = "Standard"
+  {{ if .Values.natGateway -}}
+  public_ip_address_ids   = [{{range $index, $ip := .Values.natGateway.ipAddresses}}{{if $index}},{{end}}"${data.azurerm_public_ip.nat-ip-{{ $index }}.id}"{{end}}]
+  public_ip_prefix_ids    = [{{range $index, $ipRange := .Values.natGateway.ipAddressRanges}}{{if $index}},{{end}}"${data.azurerm_public_ip_prefix.nat-iprange-prefix-{{ $index }}.id}"{{end}}]
+  {{ else -}}
   public_ip_address_ids   = ["${azurerm_public_ip.natip.id}"]
+  {{- end }}
 }
 
 resource "azurerm_subnet_nat_gateway_association" "nat-worker-subnet-association" {

charts/internal/azure-infra/values.yaml

@@ -15,6 +15,18 @@ create:
   # name: identity-name
   # resourceGroup: identity-resource-group
 
+# natGateway:
+#   ipAddresses:
+#   - name: ip-name
+#     resourceGroup: ip-resource-group
+#   - name: ip-name
+#     resourceGroup: ip-resource-group
+#   ipAddressRanges:
+#   - name: ip-range-name
+#     resourceGroup: ip-range-resource-group
+#   - name: ip-range-name
+#     resourceGroup: ip-range-resource-group
+
 resourceGroup:
   name: my-resource-group
   vnet:

docs/usage-as-end-user.md

@@ -66,7 +66,7 @@ You can freely choose this CIDR and it is your responsibility to properly design
 
 In the `networks.serviceEndpoints[]` list you can specify the list of Azure service endpoints which shall be associated with the worker subnet. All available service endpoints and their technical names can be found in the (Azure Service Endpoint documentation](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-service-endpoints-overview).
 
-The `networks.natGateway` section contains configuration for the Azure NatGateway which can be attached to the worker subnet of the Shoot cluster. The NatGateway is currently optional and can be enabled/disabled via the field `networks.natGateway.enabled`. If the NatGateway is not deployed then the outgoing traffic initiated within the Shoot cluster will be routed via cluster LoadBalancer (default behaviour, see [here](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections#scenarios)). **Restrictions:** The NatGateway is currently only available for zoned clusters (`.zoned=true`, see [#43](https://github.com/gardener/gardener-extension-provider-azure/issues/43) for more details) and it will not be deployed zone-redundant yet. Furthermore, the Azure NatGateway is not yet generally available (GA) from Azure side, hence, you need to register your subscription to participate in the preview for NatGateway.
+The `networks.natGateway` section contains configuration for the Azure NatGateway which can be attached to the worker subnet of the Shoot cluster. The NatGateway is currently optional and can be enabled/disabled via the field `networks.natGateway.enabled`. If the NatGateway is not deployed then the outgoing traffic initiated within the Shoot cluster will be routed via cluster LoadBalancer (default behaviour, see [here](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections#scenarios)). Via `networks.natGateway.ipAddresses` and `networks.natGateway.ipAddressRanges` can own public ip addresses and/or public ip ranges/prefixes defined whose should be attached to the NatGateway. For each public ip addresses or public ip range/prefix need a `.name` (name of the Azure resource) and a `.resourceGroup` (resource group which contains the Azure resource) specified. **Restrictions:** The NatGateway is currently only available for zoned clusters (`.zoned=true`, see [#43](https://github.com/gardener/gardener-extension-provider-azure/issues/43) for more details) and it will not be deployed zone-redundant yet. Furthermore, the Azure NatGateway is not yet generally available (GA) from Azure side, hence, you need to register your subscription to participate in the preview for NatGateway.
 
 Via the `.zoned` boolean you can tell whether you want to use Azure availability zones or not.
 If you don't use zones then an availability set will be created and only basic load balancers will be used.

hack/api-reference/api.md

@@ -326,6 +326,47 @@ string
 </tr>
 </tbody>
 </table>
+<h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.AzureResourceReference">AzureResourceReference
+</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#azure.provider.extensions.gardener.cloud/v1alpha1.NatGatewayConfig">NatGatewayConfig</a>)
+</p>
+<p>
+<p>AzureResourceReference contains information to identify a generic Azure resource.</p>
+</p>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>name</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>Name is the name of the referenced Azure resource.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>resourceGroup</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>ResourceGroup is the resource group which contain the referenced Azure resource.</p>
+</td>
+</tr>
+</tbody>
+</table>
 <h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.CloudControllerManagerConfig">CloudControllerManagerConfig
 </h3>
 <p>
@@ -774,6 +815,34 @@ bool
 <p>Enabled is an indicator if NAT gateway should be deployed.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>ipAddresses</code></br>
+<em>
+<a href="#azure.provider.extensions.gardener.cloud/v1alpha1.AzureResourceReference">
+[]AzureResourceReference
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>IPAddresses is a list of ip addresses which should be assigned to the NAT gateway.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>ipAddressRanges</code></br>
+<em>
+<a href="#azure.provider.extensions.gardener.cloud/v1alpha1.AzureResourceReference">
+[]AzureResourceReference
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>IPAddressRanges is a list of ip address ranges/prefixes which should be assigned to the NAT gateway.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.NetworkConfig">NetworkConfig

pkg/apis/azure/types_infrastructure.go

@@ -33,6 +33,14 @@ type InfrastructureConfig struct {
 	Zoned bool
 }
 
+// AzureResourceReference contains information to identify a generic Azure resource.
+type AzureResourceReference struct {
+	// Name is the name of the referenced Azure resource.
+	Name string
+	// ResourceGroup is the resource group which contain the referenced Azure resource.
+	ResourceGroup string
+}
+
 // ResourceGroup is azure resource group
 type ResourceGroup struct {
 	// Name is the name of the resource group
@@ -146,6 +154,10 @@ type VNetStatus struct {
 type NatGatewayConfig struct {
 	// Enabled is an indicator if NAT gateway should be deployed.
 	Enabled bool
+	// IPAddresses is a list of ip addresses which should be assigned to the NAT gateway.
+	IPAddresses []AzureResourceReference
+	// IPAddressRanges is a list of ip address ranges/prefixes which should be assigned to the NAT gateway.
+	IPAddressRanges []AzureResourceReference
 }
 
 // IdentityConfig contains configuration for the managed identity.

pkg/apis/azure/v1alpha1/types_infrastructure.go

@@ -37,6 +37,14 @@ type InfrastructureConfig struct {
 	Zoned bool `json:"zoned,omitempty"`
 }
 
+// AzureResourceReference contains information to identify a generic Azure resource.
+type AzureResourceReference struct {
+	// Name is the name of the referenced Azure resource.
+	Name string `json:"name"`
+	// ResourceGroup is the resource group which contain the referenced Azure resource.
+	ResourceGroup string `json:"resourceGroup"`
+}
+
 // ResourceGroup is azure resource group
 type ResourceGroup struct {
 	// Name is the name of the resource group
@@ -159,6 +167,12 @@ type VNetStatus struct {
 type NatGatewayConfig struct {
 	// Enabled is an indicator if NAT gateway should be deployed.
 	Enabled bool `json:"enabled"`
+	// IPAddresses is a list of ip addresses which should be assigned to the NAT gateway.
+	// +optional
+	IPAddresses []AzureResourceReference `json:"ipAddresses,omitempty"`
+	// IPAddressRanges is a list of ip address ranges/prefixes which should be assigned to the NAT gateway.
+	// +optional
+	IPAddressRanges []AzureResourceReference `json:"ipAddressRanges,omitempty"`
 }
 
 // IdentityConfig contains configuration for the managed identity.

pkg/apis/azure/validation/infrastructure.go

@@ -92,6 +92,20 @@ func ValidateInfrastructureConfig(infra *apisazure.InfrastructureConfig, nodesCI
 	if !infra.Zoned && infra.Networks.NatGateway != nil {
 		allErrs = append(allErrs, field.Invalid(fldPath.Child("networks", "natGateway"), infra.Networks.NatGateway, "NatGateway is currently only supported for zoned cluster"))
 	}
+	if infra.Zoned && infra.Networks.NatGateway != nil {
+		if !infra.Networks.NatGateway.Enabled && (len(infra.Networks.NatGateway.IPAddresses) > 0 || len(infra.Networks.NatGateway.IPAddressRanges) > 0) {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("networks", "natGateway"), infra.Networks.NatGateway, "NatGateway is not enabled but ip addresses or ip ranges are specified"))
+		} else {
+			ipAddressesField := fldPath.Child("networks", "natGateway", "ipAddresses")
+			for i, ip := range infra.Networks.NatGateway.IPAddresses {
+				allErrs = append(allErrs, validateAzureResourceReference(&ip, ipAddressesField.Index(i))...)
+			}
+			ipAddressRangesField := fldPath.Child("networks", "natGateway", "ipAddressRanges")
+			for i, ipRange := range infra.Networks.NatGateway.IPAddressRanges {
+				allErrs = append(allErrs, validateAzureResourceReference(&ipRange, ipAddressRangesField.Index(i))...)
+			}
+		}
+	}
 
 	if infra.Identity != nil && (infra.Identity.Name == "" || infra.Identity.ResourceGroup == "") {
 		allErrs = append(allErrs, field.Invalid(fldPath.Child("identity"), infra.Identity, "specifying an identity requires the name of the identity and the resource group which hosts the identity"))
@@ -104,6 +118,17 @@ func ValidateInfrastructureConfig(infra *apisazure.InfrastructureConfig, nodesCI
 	return allErrs
 }
 
+func validateAzureResourceReference(ref *apisazure.AzureResourceReference, fieldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if ref.Name == "" {
+		allErrs = append(allErrs, field.Invalid(fieldPath, ref.Name, "name must be set"))
+	}
+	if ref.ResourceGroup == "" {
+		allErrs = append(allErrs, field.Invalid(fieldPath, ref.ResourceGroup, "resourceGroup must be set"))
+	}
+	return allErrs
+}
+
 // ValidateInfrastructureConfigUpdate validates a InfrastructureConfig object.
 func ValidateInfrastructureConfigUpdate(oldConfig, newConfig *apisazure.InfrastructureConfig, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}