Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add blue green upgrade settings. #12984

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changelog/6618.txt
@@ -0,0 +1,6 @@
```release-note:enhancement
container: added field `strategy` to `google_container_node_pool`
```
```release-note:enhancement
container: added field `blue_green_settings` to `google_container_node_pool`
```
208 changes: 194 additions & 14 deletions google/resource_container_node_pool.go
Expand Up @@ -74,6 +74,53 @@ func resourceContainerNodePool() *schema.Resource {
}
}

var schemaBlueGreenSettings = &schema.Schema{
Type: schema.TypeList,
Optional: true,
Computed: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"standard_rollout_policy": {
Type: schema.TypeList,
Required: true,
MaxItems: 1,
Description: `Standard rollout policy is the default policy for blue-green.`,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"batch_percentage": {
Type: schema.TypeFloat,
Optional: true,
Computed: true,
Description: `Percentage of the blue pool nodes to drain in a batch.`,
ValidateFunc: validation.FloatBetween(0.0, 1.0),
},
"batch_node_count": {
Type: schema.TypeInt,
Optional: true,
Computed: true,
Description: `Number of blue nodes to drain in a batch.`,
},
"batch_soak_duration": {
Type: schema.TypeString,
Optional: true,
Computed: true,
Description: `Soak time after each batch gets drained.`,
},
},
},
},
"node_pool_soak_duration": {
Type: schema.TypeString,
Optional: true,
Computed: true,
Description: `Time needed after draining entire blue pool. After this period, blue pool will be cleaned up.`,
},
},
},
Description: `Settings for BlueGreen node pool upgrade.`,
}

var schemaNodePool = map[string]*schema.Schema{
"autoscaling": {
Type: schema.TypeList,
Expand Down Expand Up @@ -146,17 +193,29 @@ var schemaNodePool = map[string]*schema.Schema{
Schema: map[string]*schema.Schema{
"max_surge": {
Type: schema.TypeInt,
Required: true,
Optional: true,
Computed: true,
ValidateFunc: validation.IntAtLeast(0),
Description: `The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater.`,
},

"max_unavailable": {
Type: schema.TypeInt,
Required: true,
Optional: true,
Computed: true,
ValidateFunc: validation.IntAtLeast(0),
Description: `The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater.`,
},

"strategy": {
Type: schema.TypeString,
Optional: true,
Default: "SURGE",
ValidateFunc: validation.StringInSlice([]string{"SURGE", "BLUE_GREEN"}, false),
Description: `Update strategy for the given nodepool.`,
},

"blue_green_settings": schemaBlueGreenSettings,
},
},
},
Expand Down Expand Up @@ -749,18 +808,103 @@ func expandNodePool(d *schema.ResourceData, prefix string) (*container.NodePool,
upgradeSettingsConfig := v.([]interface{})[0].(map[string]interface{})
np.UpgradeSettings = &container.UpgradeSettings{}

if v, ok := upgradeSettingsConfig["max_surge"]; ok {
np.UpgradeSettings.MaxSurge = int64(v.(int))
if v, ok := upgradeSettingsConfig["strategy"]; ok {
np.UpgradeSettings.Strategy = v.(string)
}

if v, ok := upgradeSettingsConfig["max_unavailable"]; ok {
np.UpgradeSettings.MaxUnavailable = int64(v.(int))
if d.HasChange(prefix + "upgrade_settings.0.max_surge") {
if np.UpgradeSettings.Strategy != "SURGE" {
return nil, fmt.Errorf("Surge upgrade settings may not be changed when surge strategy is not enabled")
}
if v, ok := upgradeSettingsConfig["max_surge"]; ok {
np.UpgradeSettings.MaxSurge = int64(v.(int))
}
}

if d.HasChange(prefix + "upgrade_settings.0.max_unavailable") {
if np.UpgradeSettings.Strategy != "SURGE" {
return nil, fmt.Errorf("Surge upgrade settings may not be changed when surge strategy is not enabled")
}
if v, ok := upgradeSettingsConfig["max_unavailable"]; ok {
np.UpgradeSettings.MaxUnavailable = int64(v.(int))
}
}

if v, ok := upgradeSettingsConfig["blue_green_settings"]; ok && len(v.([]interface{})) > 0 {
blueGreenSettingsConfig := v.([]interface{})[0].(map[string]interface{})
np.UpgradeSettings.BlueGreenSettings = &container.BlueGreenSettings{}

if np.UpgradeSettings.Strategy != "BLUE_GREEN" {
return nil, fmt.Errorf("Blue-green upgrade settings may not be changed when blue-green strategy is not enabled")
}

if v, ok := blueGreenSettingsConfig["node_pool_soak_duration"]; ok {
np.UpgradeSettings.BlueGreenSettings.NodePoolSoakDuration = v.(string)
}

if v, ok := blueGreenSettingsConfig["standard_rollout_policy"]; ok && len(v.([]interface{})) > 0 {
standardRolloutPolicyConfig := v.([]interface{})[0].(map[string]interface{})
standardRolloutPolicy := &container.StandardRolloutPolicy{}

if v, ok := standardRolloutPolicyConfig["batch_soak_duration"]; ok {
standardRolloutPolicy.BatchSoakDuration = v.(string)
}
if v, ok := standardRolloutPolicyConfig["batch_node_count"]; ok {
standardRolloutPolicy.BatchNodeCount = int64(v.(int))
}
if v, ok := standardRolloutPolicyConfig["batch_percentage"]; ok {
standardRolloutPolicy.BatchPercentage = v.(float64)
}

np.UpgradeSettings.BlueGreenSettings.StandardRolloutPolicy = standardRolloutPolicy
}
}
}

return np, nil
}

func flattenNodePoolStandardRolloutPolicy(rp *container.StandardRolloutPolicy) []map[string]interface{} {
if rp == nil {
return nil
}

return []map[string]interface{}{
{
"batch_node_count": rp.BatchNodeCount,
"batch_percentage": rp.BatchPercentage,
"batch_soak_duration": rp.BatchSoakDuration,
},
}
}

func flattenNodePoolBlueGreenSettings(bg *container.BlueGreenSettings) []map[string]interface{} {
if bg == nil {
return nil
}
return []map[string]interface{}{
{
"node_pool_soak_duration": bg.NodePoolSoakDuration,
"standard_rollout_policy": flattenNodePoolStandardRolloutPolicy(bg.StandardRolloutPolicy),
},
}
}

func flattenNodePoolUpgradeSettings(us *container.UpgradeSettings) []map[string]interface{} {
if us == nil {
return nil
}

upgradeSettings := make(map[string]interface{})

upgradeSettings["blue_green_settings"] = flattenNodePoolBlueGreenSettings(us.BlueGreenSettings)
upgradeSettings["max_surge"] = us.MaxSurge
upgradeSettings["max_unavailable"] = us.MaxUnavailable

upgradeSettings["strategy"] = us.Strategy
return []map[string]interface{}{upgradeSettings}
}

func flattenNodePool(d *schema.ResourceData, config *Config, np *container.NodePool, prefix string) (map[string]interface{}, error) {
userAgent, err := generateUserAgentString(d, config.userAgent)
if err != nil {
Expand Down Expand Up @@ -835,12 +979,7 @@ func flattenNodePool(d *schema.ResourceData, config *Config, np *container.NodeP
}

if np.UpgradeSettings != nil {
nodePool["upgrade_settings"] = []map[string]interface{}{
{
"max_surge": np.UpgradeSettings.MaxSurge,
"max_unavailable": np.UpgradeSettings.MaxUnavailable,
},
}
nodePool["upgrade_settings"] = flattenNodePoolUpgradeSettings(np.UpgradeSettings)
} else {
delete(nodePool, "upgrade_settings")
}
Expand Down Expand Up @@ -1165,8 +1304,49 @@ func nodePoolUpdate(d *schema.ResourceData, meta interface{}, nodePoolInfo *Node
upgradeSettings := &container.UpgradeSettings{}
if v, ok := d.GetOk(prefix + "upgrade_settings"); ok {
upgradeSettingsConfig := v.([]interface{})[0].(map[string]interface{})
upgradeSettings.MaxSurge = int64(upgradeSettingsConfig["max_surge"].(int))
upgradeSettings.MaxUnavailable = int64(upgradeSettingsConfig["max_unavailable"].(int))
upgradeSettings.Strategy = upgradeSettingsConfig["strategy"].(string)

if d.HasChange(prefix + "upgrade_settings.0.max_surge") {
if upgradeSettings.Strategy != "SURGE" {
return fmt.Errorf("Surge upgrade settings may not be changed when surge strategy is not enabled")
}
if v, ok := upgradeSettingsConfig["max_surge"]; ok {
upgradeSettings.MaxSurge = int64(v.(int))
}
}

if d.HasChange(prefix + "upgrade_settings.0.max_unavailable") {
if upgradeSettings.Strategy != "SURGE" {
return fmt.Errorf("Surge upgrade settings may not be changed when surge strategy is not enabled")
}
if v, ok := upgradeSettingsConfig["max_unavailable"]; ok {
upgradeSettings.MaxUnavailable = int64(v.(int))
}
}

if d.HasChange(prefix + "upgrade_settings.0.blue_green_settings") {
if upgradeSettings.Strategy != "BLUE_GREEN" {
return fmt.Errorf("Blue-green upgrade settings may not be changed when blue-green strategy is not enabled")
}

blueGreenSettings := &container.BlueGreenSettings{}
blueGreenSettingsConfig := upgradeSettingsConfig["blue_green_settings"].([]interface{})[0].(map[string]interface{})
blueGreenSettings.NodePoolSoakDuration = blueGreenSettingsConfig["node_pool_soak_duration"].(string)

if v, ok := blueGreenSettingsConfig["standard_rollout_policy"]; ok && len(v.([]interface{})) > 0 {
standardRolloutPolicy := &container.StandardRolloutPolicy{}
standardRolloutPolicyConfig := v.([]interface{})[0].(map[string]interface{})
standardRolloutPolicy.BatchSoakDuration = standardRolloutPolicyConfig["batch_soak_duration"].(string)
if v, ok := standardRolloutPolicyConfig["batch_node_count"]; ok {
standardRolloutPolicy.BatchNodeCount = int64(v.(int))
}
if v, ok := standardRolloutPolicyConfig["batch_percentage"]; ok {
standardRolloutPolicy.BatchPercentage = v.(float64)
}
blueGreenSettings.StandardRolloutPolicy = standardRolloutPolicy
}
upgradeSettings.BlueGreenSettings = blueGreenSettings
}
}
req := &container.UpdateNodePoolRequest{
UpgradeSettings: upgradeSettings,
Expand Down
54 changes: 46 additions & 8 deletions google/resource_container_node_pool_test.go
Expand Up @@ -306,15 +306,31 @@ func TestAccContainerNodePool_withUpgradeSettings(t *testing.T) {
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccContainerNodePool_withUpgradeSettings(cluster, np, 2, 3),
Config: testAccContainerNodePool_withUpgradeSettings(cluster, np, 2, 3, "SURGE", "", 0, 0.0, ""),
},
{
ResourceName: "google_container_node_pool.with_upgrade_settings",
ImportState: true,
ImportStateVerify: true,
},
{
Config: testAccContainerNodePool_withUpgradeSettings(cluster, np, 1, 1),
Config: testAccContainerNodePool_withUpgradeSettings(cluster, np, 1, 1, "SURGE", "", 0, 0.0, ""),
},
{
ResourceName: "google_container_node_pool.with_upgrade_settings",
ImportState: true,
ImportStateVerify: true,
},
{
Config: testAccContainerNodePool_withUpgradeSettings(cluster, np, 0, 0, "BLUE_GREEN", "100s", 1, 0.0, "0s"),
},
{
ResourceName: "google_container_node_pool.with_upgrade_settings",
ImportState: true,
ImportStateVerify: true,
},
{
Config: testAccContainerNodePool_withUpgradeSettings(cluster, np, 0, 0, "BLUE_GREEN", "100s", 0, 0.5, "1s"),
},
{
ResourceName: "google_container_node_pool.with_upgrade_settings",
Expand Down Expand Up @@ -1572,7 +1588,32 @@ resource "google_container_node_pool" "with_workload_metadata_config" {
`, projectID, cluster, np)
}

func testAccContainerNodePool_withUpgradeSettings(clusterName string, nodePoolName string, maxSurge int, maxUnavailable int) string {
func makeUpgradeSettings(maxSurge int, maxUnavailable int, strategy string, nodePoolSoakDuration string, batchNodeCount int, batchPercentage float64, batchSoakDuration string) string {
if strategy == "BLUE_GREEN" {
return fmt.Sprintf(`
upgrade_settings {
strategy = "%s"
blue_green_settings {
node_pool_soak_duration = "%s"
standard_rollout_policy {
batch_node_count = %d
batch_percentage = %f
batch_soak_duration = "%s"
}
}
}
`, strategy, nodePoolSoakDuration, batchNodeCount, batchPercentage, batchSoakDuration)
}
return fmt.Sprintf(`
upgrade_settings {
max_surge = %d
max_unavailable = %d
strategy = "%s"
}
`, maxSurge, maxUnavailable, strategy)
}

func testAccContainerNodePool_withUpgradeSettings(clusterName string, nodePoolName string, maxSurge int, maxUnavailable int, strategy string, nodePoolSoakDuration string, batchNodeCount int, batchPercentage float64, batchSoakDuration string) string {
return fmt.Sprintf(`
data "google_container_engine_versions" "central1" {
location = "us-central1"
Expand All @@ -1590,12 +1631,9 @@ resource "google_container_node_pool" "with_upgrade_settings" {
location = "us-central1"
cluster = "${google_container_cluster.cluster.name}"
initial_node_count = 1
upgrade_settings {
max_surge = %d
max_unavailable = %d
}
%s
}
`, clusterName, nodePoolName, maxSurge, maxUnavailable)
`, clusterName, nodePoolName, makeUpgradeSettings(maxSurge, maxUnavailable, strategy, nodePoolSoakDuration, batchNodeCount, batchPercentage, batchSoakDuration))
}

func testAccContainerNodePool_withGPU(cluster, np string) string {
Expand Down
22 changes: 18 additions & 4 deletions website/docs/r/container_node_pool.html.markdown
Expand Up @@ -157,8 +157,7 @@ cluster.
* `project` - (Optional) The ID of the project in which to create the node pool. If blank,
the provider-configured project will be used.

* `upgrade_settings` (Optional) Specify node upgrade settings to change how many nodes GKE attempts to
upgrade at once. The number of nodes upgraded simultaneously is the sum of `max_surge` and `max_unavailable`.
* `upgrade_settings` (Optional) Specify node upgrade settings to change how GKE upgrades nodes.
The maximum number of nodes upgraded simultaneously is limited to 20. Structure is [documented below](#nested_upgrade_settings).

* `version` - (Optional) The Kubernetes version for the nodes in this pool. Note that if this field
Expand Down Expand Up @@ -201,16 +200,31 @@ cluster.

<a name="nested_upgrade_settings"></a>The `upgrade_settings` block supports:

* `max_surge` - (Required) The number of additional nodes that can be added to the node pool during
* `max_surge` - (Optional) The number of additional nodes that can be added to the node pool during
an upgrade. Increasing `max_surge` raises the number of nodes that can be upgraded simultaneously.
Can be set to 0 or greater.

* `max_unavailable` - (Required) The number of nodes that can be simultaneously unavailable during
* `max_unavailable` - (Optional) The number of nodes that can be simultaneously unavailable during
an upgrade. Increasing `max_unavailable` raises the number of nodes that can be upgraded in
parallel. Can be set to 0 or greater.

`max_surge` and `max_unavailable` must not be negative and at least one of them must be greater than zero.

* `strategy` - (Default `SURGE`) The upgrade stragey to be used for upgrading the nodes.

* `blue_green_settings` - (Optional) The settings to adjust [blue green upgrades](https://cloud.google.com/kubernetes-engine/docs/concepts/node-pool-upgrade-strategies#blue-green-upgrade-strategy).
Structure is [documented below](#nested_blue_green_settings)

<a name="nested_blue_green_settings"></a>The `blue_green_settings` block supports:

* `standard_rollout_policy` - (Required) Specifies the standard policy settings for blue-green upgrades.
* `batch_percentage` - (Optional) Percentage of the blue pool nodes to drain in a batch.
* `batch_node_count` - (Optional) Number of blue nodes to drain in a batch.
* `batch_soak_duration` - (Optionial) Soak time after each batch gets drained.

* `node_pool_soak_duration` - (Optional) Time needed after draining the entire blue pool.
After this period, the blue pool will be cleaned up.

<a name="nested_placement_policy"></a>The `placement_policy` block supports:

* `type` - (Required) The type of the policy. Supports a single value: COMPACT.
Expand Down