NatGateway Integration – Step 2

charts/internal/azure-infra/templates/_helpers.tpl

@@ -0,0 +1,39 @@
+{{- define "resource-group-reference" -}}
+{{- if .Values.create.resourceGroup -}}
+azurerm_resource_group.rg.name
+{{- else -}}
+data.azurerm_resource_group.rg.name
+{{- end}}
+{{- end -}}
+
+{{- define "natgateway-managed-ip" -}}
+# Gardener managed public IP to be attached to the NatGateway.
+resource "azurerm_public_ip" "natip" {
+  name                = "{{ required "clusterName is required" .Values.clusterName }}-nat-ip"
+  location            = "{{ required "azure.region is required" .Values.azure.region }}"
+  resource_group_name = {{ template "resource-group-reference" . }}
+  allocation_method   = "Static"
+  sku                 = "Standard"
+  {{ if .Values.natGateway -}}{{ if hasKey .Values.natGateway "zone" -}}
+  zones = [{{ .Values.natGateway.zone | quote }}]
+  {{- end }}{{- end }}
+}
+resource "azurerm_nat_gateway_public_ip_association" "natip-association" {
+  nat_gateway_id       = azurerm_nat_gateway.nat.id
+  public_ip_address_id = azurerm_public_ip.natip.id
+}
+{{- end -}}
+
+{{- define "natgateway-user-provided-public-ips" -}}
+# User provided public IPs to be attached to the NatGateway.
+{{ range $index, $ip := .Values.natGateway.ipAddresses -}}
+data "azurerm_public_ip" "natip-user-provided-{{ $index }}" {
+  name                = "{{ $ip.name }}"
+  resource_group_name = "{{ $ip.resourceGroup }}"
+}
+resource "azurerm_nat_gateway_public_ip_association" "natip-user-provided-{{ $index }}-association" {
+  nat_gateway_id       = azurerm_nat_gateway.nat.id
+  public_ip_address_id = data.azurerm_public_ip.natip-user-provided-{{ $index }}.id
+}
+{{ end }}
+{{- end -}}

charts/internal/azure-infra/templates/main.tf

@@ -7,6 +7,9 @@ provider "azurerm" {
   features {}
 }
 
+#===============================================
+#= Resource Group
+#===============================================
 {{ if .Values.create.resourceGroup -}}
 resource "azurerm_resource_group" "rg" {
   name     = "{{ required "resourceGroup.name is required" .Values.resourceGroup.name }}"
@@ -22,14 +25,11 @@ data "azurerm_resource_group" "rg" {
 #= VNet, Subnets, Route Table, Security Groups
 #===============================================
 
+# VNet
 {{ if .Values.create.vnet -}}
 resource "azurerm_virtual_network" "vnet" {
   name                = "{{ required "resourceGroup.vnet.name is required" .Values.resourceGroup.vnet.name }}"
-  {{ if .Values.create.resourceGroup -}}
-  resource_group_name = azurerm_resource_group.rg.name
-  {{- else -}}
-  resource_group_name = data.azurerm_resource_group.rg.name
-  {{- end}}
+  resource_group_name = {{ template "resource-group-reference" . }}
   location            = "{{ required "azure.region is required" .Values.azure.region }}"
   address_space       = ["{{ required "resourceGroup.vnet.cidr is required" .Values.resourceGroup.vnet.cidr }}"]
 }
@@ -40,6 +40,7 @@ data "azurerm_virtual_network" "vnet" {
 }
 {{- end }}
 
+# Subnet
 resource "azurerm_subnet" "workers" {
   name                      = "{{ required "clusterName is required" .Values.clusterName }}-nodes"
   {{ if .Values.create.vnet -}}
@@ -53,31 +54,23 @@ resource "azurerm_subnet" "workers" {
   service_endpoints         = [{{range $index, $serviceEndpoint := .Values.resourceGroup.subnet.serviceEndpoints}}{{if $index}},{{end}}"{{$serviceEndpoint}}"{{end}}]
 }
 
+# RouteTable
 resource "azurerm_route_table" "workers" {
   name                = "worker_route_table"
   location            = "{{ required "azure.region is required" .Values.azure.region }}"
-  {{ if .Values.create.resourceGroup -}}
-  resource_group_name = azurerm_resource_group.rg.name
-  {{- else -}}
-  resource_group_name = data.azurerm_resource_group.rg.name
-  {{- end}}
+  resource_group_name = {{ template "resource-group-reference" . }}
 }
-
 resource "azurerm_subnet_route_table_association" "workers-rt-subnet-association" {
   subnet_id      = azurerm_subnet.workers.id
   route_table_id = azurerm_route_table.workers.id
 }
 
+# SecurityGroup
 resource "azurerm_network_security_group" "workers" {
   name                = "{{ required "clusterName is required" .Values.clusterName }}-workers"
   location            = "{{ required "azure.region is required" .Values.azure.region }}"
-  {{ if .Values.create.resourceGroup -}}
-  resource_group_name = azurerm_resource_group.rg.name
-  {{- else -}}
-  resource_group_name = data.azurerm_resource_group.rg.name
-  {{- end}}
+  resource_group_name = {{ template "resource-group-reference" . }}
 }
-
 resource "azurerm_subnet_network_security_group_association" "workers-nsg-subnet-association" {
   subnet_id                 = azurerm_subnet.workers.id
   network_security_group_id = azurerm_network_security_group.workers.id
@@ -88,48 +81,38 @@ resource "azurerm_subnet_network_security_group_association" "workers-nsg-subnet
 #= NAT Gateway
 #===============================================
 
-resource "azurerm_public_ip" "natip" {
-  name                = "{{ required "clusterName is required" .Values.clusterName }}-nat-ip"
-  location            = "{{ required "azure.region is required" .Values.azure.region }}"
-  {{ if .Values.create.resourceGroup -}}
-  resource_group_name = azurerm_resource_group.rg.name
-  {{- else -}}
-  resource_group_name = data.azurerm_resource_group.rg.name
-  {{- end }}
-  allocation_method   = "Static"
-  sku                 = "Standard"
-}
-
 resource "azurerm_nat_gateway" "nat" {
   name                    = "{{ required "clusterName is required" .Values.clusterName }}-nat-gateway"
   location                = "{{ required "azure.region is required" .Values.azure.region }}"
-  {{ if .Values.create.resourceGroup -}}
-  resource_group_name     = azurerm_resource_group.rg.name
-  {{- else -}}
-  resource_group_name     = data.azurerm_resource_group.rg.name
-  {{- end }}
+  resource_group_name     = {{ template "resource-group-reference" . }}
   sku_name                = "Standard"
   {{ if .Values.natGateway -}}
-  {{ if .Values.natGateway.idleConnectionTimeoutMinutes -}}
+  {{ if hasKey .Values.natGateway "idleConnectionTimeoutMinutes" -}}
   idle_timeout_in_minutes = {{ .Values.natGateway.idleConnectionTimeoutMinutes }}
   {{- end }}
-
-  # TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
+  {{ if hasKey .Values.natGateway "zone" -}}
+  zones = [{{ .Values.natGateway.zone | quote }}]
+  {{- end }}
   {{ if .Values.natGateway.migrateNatGatewayToIPAssociation -}}
+  # TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
   public_ip_address_ids   = []
   {{- end }}
   {{- end }}
 }
-
-resource "azurerm_nat_gateway_public_ip_association" "natip-association" {
-  nat_gateway_id       = azurerm_nat_gateway.nat.id
-  public_ip_address_id = azurerm_public_ip.natip.id
-}
-
 resource "azurerm_subnet_nat_gateway_association" "nat-worker-subnet-association" {
   subnet_id      = azurerm_subnet.workers.id
   nat_gateway_id = azurerm_nat_gateway.nat.id
 }
+
+{{ if .Values.natGateway -}}
+{{ if and (hasKey .Values.natGateway "ipAddresses") (hasKey .Values.natGateway "zone") -}}
+{{ template "natgateway-user-provided-public-ips" . }}
+{{- else -}}
+{{ template "natgateway-managed-ip" . }}
+{{- end }}
+{{- else -}}
+{{ template "natgateway-managed-ip" . }}
+{{- end }}
 {{- end }}
 
 {{ if .Values.identity -}}
@@ -150,12 +133,8 @@ data "azurerm_user_assigned_identity" "identity" {
 
 resource "azurerm_availability_set" "workers" {
   name                         = "{{ required "clusterName is required" .Values.clusterName }}-avset-workers"
-  {{ if .Values.create.resourceGroup -}}
-  resource_group_name          = azurerm_resource_group.rg.name
-  {{- else -}}
-  resource_group_name          = data.azurerm_resource_group.rg.name
-  {{- end}}
   location                     = "{{ required "azure.region is required" .Values.azure.region }}"
+  resource_group_name          = {{ template "resource-group-reference" . }}
   platform_update_domain_count = "{{ required "azure.countUpdateDomains is required" .Values.azure.countUpdateDomains }}"
   platform_fault_domain_count  = "{{ required "azure.countFaultDomains is required" .Values.azure.countFaultDomains }}"
   managed                      = true
@@ -167,11 +146,7 @@ resource "azurerm_availability_set" "workers" {
 #===============================================
 
 output "{{ .Values.outputKeys.resourceGroupName }}" {
-{{- if .Values.create.resourceGroup }}
-  value = azurerm_resource_group.rg.name
-{{- else -}}
-  value = data.azurerm_resource_group.rg.name
-{{- end}}
+  value = {{ template "resource-group-reference" . }}
 }
 
 {{ if .Values.create.vnet -}}

charts/internal/azure-infra/values.yaml

@@ -15,10 +15,15 @@ create:
   # name: identity-name
   # resourceGroup: identity-resource-group
 
-natGateway:
-  idleConnectionTimeoutMinutes:
-  # TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
-  migrateNatGatewayToIPAssociation: false
+# natGateway:
+  # idleConnectionTimeoutMinutes: 4
+  # zone: 1
+  # migrateNatGatewayToIPAssociation: false # TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
+  # ipAddresses:
+  # - name: public-ip-0-name
+  #   resourceGroup: public-ip-0-resource-group
+  # - name: public-ip-1-name
+  #   resourceGroup: public-ip-1-resource-group
 
 resourceGroup:
   name: my-resource-group

docs/usage-as-end-user.md

@@ -51,6 +51,11 @@ networks:
   # natGateway:
   #   enabled: false
   #   idleConnectionTimeoutMinutes: 4
+  #   zone: 1
+  #   ipAddresses:
+  #   - name: my-public-ip-name
+  #     resourceGroup: my-public-ip-resource-group
+  #     zone: 1
   # serviceEndpoints:
   # - Microsoft.Test
 zoned: false
@@ -62,6 +67,13 @@ zoned: false
 #  acrAccess: true
 ```
 
+Currently, it's not yet possible to deploy into existing resource groups, but in the future it will.
+The `.resourceGroup.name` field will allow specifying the name of an already existing resource group that the shoot cluster and all infrastructure resources will be deployed to.
+
+Via the `.zoned` boolean you can tell whether you want to use Azure availability zones or not.
+If you don't use zones then an availability set will be created and only basic load balancers will be used.
+Zoned clusters use standard load balancers.
+
 The `networks.vnet` section describes whether you want to create the shoot cluster in an already existing VNet or whether to create a new one:
 
 * If `networks.vnet.name` and `networks.vnet.resourceGroup` are given then you have to specify the VNet name and VNet resource group name of the existing VNet that was created by other means (manually, other tooling, ...).
@@ -75,18 +87,17 @@ You can freely choose this CIDR and it is your responsibility to properly design
 
 In the `networks.serviceEndpoints[]` list you can specify the list of Azure service endpoints which shall be associated with the worker subnet. All available service endpoints and their technical names can be found in the (Azure Service Endpoint documentation](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-service-endpoints-overview).
 
-The `networks.natGateway` section contains configuration for the Azure NatGateway which can be attached to the worker subnet of the Shoot cluster. The NatGateway is currently optional and can be enabled/disabled via the field `networks.natGateway.enabled`. If the NatGateway is not deployed then the outgoing traffic initiated within the Shoot cluster will be routed via cluster LoadBalancer (default behaviour, see [here](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections#scenarios)). **Restrictions:** The NatGateway is currently only available for zoned clusters (`.zoned=true`, see [#43](https://github.com/gardener/gardener-extension-provider-azure/issues/43) for more details) and it will not be deployed zone-redundant yet. If NAT Gateway is enabled, the field `networks.natGateway.idleConnectionTimeoutMinutes` allows the configuration of NAT Gateway's idle connection timeout property. The idle timeout value can be adjusted from 4 minutes, up to 120 minutes. Omitting this property will set the idle timeout to its default value according to [NAT Gateway's documentation](https://docs.microsoft.com/en-us/azure/virtual-network/nat-gateway-resource#timers).
-
-Via the `.zoned` boolean you can tell whether you want to use Azure availability zones or not.
-If you don't use zones then an availability set will be created and only basic load balancers will be used.
-Zoned clusters use standard load balancers.
+The `networks.natGateway` section contains configuration for the Azure NatGateway which can be attached to the worker subnet of a Shoot cluster. Here are some key information about the usage of the NatGateway for a Shoot cluster:
+- NatGateway usage is optional and can be enabled or disabled via `.networks.natGateway.enabled`.
+- If the NatGateway is not used then the egress connections initiated within the Shoot cluster will be nated via the LoadBalancer of the clusters (default Azure behaviour, see [here](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections#scenarios)).
+- NatGateway is only available for zonal clusters `.zoned=true`.
+- The NatGateway is currently **not** zone redundantly deployed. That mean the NatGateway of a Shoot cluster will always be in just one zone. This zone can be optionally selected via `.networks.natGateway.zone`.
+- **Caution:** Modifying the `.networks.natGateway.zone` setting requires a recreation of the NatGateway and the managed public ip (automatically used if no own public ip is specified, see below). That mean you will most likely get a different public ip for egress connections.
+- It is possible to bring own zonal public ip(s) via `networks.natGateway.ipAddresses`. Those public ip(s) need to be in the same zone as the NatGateway (see `networks.natGateway.zone`) and be of SKU `standard`. For each public the `name`, the `resourceGroup` and the `zone` need to be specified.
+- The field `networks.natGateway.idleConnectionTimeoutMinutes` allows the configuration of NAT Gateway's idle connection timeout property. The idle timeout value can be adjusted from 4 minutes, up to 120 minutes. Omitting this property will set the idle timeout to its default value according to [NAT Gateway's documentation](https://docs.microsoft.com/en-us/azure/virtual-network/nat-gateway-resource#timers).
 
 In the `identity` section you can specify an [Azure user-assigned managed identity](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/overview#how-does-the-managed-identities-for-azure-resources-work) which should be attached to all cluster worker machines. With `identity.name` you can specify the name of the identity and with `identity.resourceGroup` you can specify the resource group which contains the identity resource on Azure. The identity need to be created by the user upfront (manually, other tooling, ...). Gardener/Azure Extension will only use the referenced one and won't create an identity. Furthermore the identity have to be in the same subscription as the Shoot cluster. Via the `identity.acrAccess` you can configure the worker machines to use the passed identity for pulling from an [Azure Container Registry (ACR)](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-intro).
-
-Adding, exchanging or removing the identity will require a rolling update of all worker machines in the Shoot cluster.
-
-Currently, it's not yet possible to deploy into existing resource groups, but in the future it will.
-The `.resourceGroup.name` field will allow specifying the name of an already existing resource group that the shoot cluster and all infrastructure resources will be deployed to.
+**Caution:** Adding, exchanging or removing the identity will require a rolling update of all worker machines in the Shoot cluster.
 
 Apart from the VNet and the worker subnet the Azure extension will also create a dedicated resource group, route tables, security groups, and an availability set (if not using zoned clusters).
 

hack/api-reference/api.md

@@ -909,7 +909,7 @@ bool
 <a href="#azure.provider.extensions.gardener.cloud/v1alpha1.NetworkConfig">NetworkConfig</a>)
 </p>
 <p>
-<p>NatGatewayConfig contains configuration for the nat gateway and the attached resources.</p>
+<p>NatGatewayConfig contains configuration for the NAT gateway and the attached resources.</p>
 </p>
 <table>
 <thead>
@@ -942,6 +942,32 @@ int32
 <p>IdleConnectionTimeoutMinutes specifies the idle connection timeout limit for NAT gateway in minutes.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>zone</code></br>
+<em>
+int32
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Zone specifies the zone in which the NAT gateway should be deployed to.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>ipAddresses</code></br>
+<em>
+<a href="#azure.provider.extensions.gardener.cloud/v1alpha1.PublicIPReference">
+[]PublicIPReference
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>IPAddresses is a list of ip addresses which should be assigned to the NAT gateway.</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.NetworkConfig">NetworkConfig
@@ -1058,6 +1084,58 @@ VNetStatus
 </tr>
 </tbody>
 </table>
+<h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.PublicIPReference">PublicIPReference
+</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#azure.provider.extensions.gardener.cloud/v1alpha1.NatGatewayConfig">NatGatewayConfig</a>)
+</p>
+<p>
+<p>PublicIPReference contains information about a public ip.</p>
+</p>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>name</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>Name is the name of the public ip.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>resourceGroup</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>ResourceGroup is the name of the resource group where the public ip is assigned to.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>zone</code></br>
+<em>
+int32
+</em>
+</td>
+<td>
+<p>Zone is the zone in which the public ip is deployed to.</p>
+</td>
+</tr>
+</tbody>
+</table>
 <h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.Purpose">Purpose
 (<code>string</code> alias)</p></h3>
 <p>