new GCP attributes for clusters & instance pools (databricks#1126)

Author: alexott

CHANGELOG.md

@@ -5,6 +5,7 @@
 * Added `workspace_url` attribute to the `databricks_current_user` data source ([#1107](https://github.com/databrickslabs/terraform-provider-databricks/pull/1107)).
 * Fixed issue at `databricks_mount` where new cluster was created for S3 mount even when `cluster_id` was specified ([#1064](https://github.com/databrickslabs/terraform-provider-databricks/issues/1064)).
 * Allow to disable auto-termination for Databricks SQL endpoints ([#900](https://github.com/databrickslabs/terraform-provider-databricks/pull/900)).
+* Added new `gcp_attributes` to `databricks_cluster` and `databricks_instance_pool` ([#1126](https://github.com/databrickslabs/terraform-provider-databricks/pull/1126)).
 
 ## 0.4.9
 

clusters/clusters_api.go

@@ -42,6 +42,17 @@ const (
 	AzureAvailabilitySpotWithFallback = "SPOT_WITH_FALLBACK_AZURE"
 )
 
+// https://docs.gcp.databricks.com/dev-tools/api/latest/clusters.html#gcpavailability
+const (
+	// GcpAvailabilityPreemptible is Preemptible instance type for clusters
+	GcpAvailabilityPreemptible = "PREEMPTIBLE_GCP"
+	// GcpAvailabilityOnDemand is OnDemand instance type for clusters
+	GcpAvailabilityOnDemand = "ON_DEMAND_GCP"
+	// GcpAvailabilityPreemptible is Preemptible instance type for clusters with option
+	// to fallback into on-demand if instance cannot be acquired
+	GcpAvailabilityPreemptibleWithFallback = "PREEMPTIBLE_WITH_FALLBACK_GCP"
+)
+
 // AzureDiskVolumeType is disk type on azure vms
 type AzureDiskVolumeType string
 
@@ -151,8 +162,11 @@ type AzureAttributes struct {
 // GcpAttributes encapsultes GCP specific attributes
 // https://docs.gcp.databricks.com/dev-tools/api/latest/clusters.html#clustergcpattributes
 type GcpAttributes struct {
-	UsePreemptibleExecutors bool   `json:"use_preemptible_executors,omitempty"`
-	GoogleServiceAccount    string `json:"google_service_account,omitempty"`
+	UsePreemptibleExecutors bool         `json:"use_preemptible_executors,omitempty"`
+	GoogleServiceAccount    string       `json:"google_service_account,omitempty"`
+	Availability            Availability `json:"availability,omitempty"`
+	BootDiskSize            int32        `json:"boot_disk_size,omitempty"`
+	ZoneId                  string       `json:"zone_id,omitempty"`
 }
 
 // DbfsStorageInfo contains the destination string for DBFS

docs/resources/cluster.md

@@ -317,7 +317,7 @@ The following options are available:
 
 `azure_attributes` optional configuration block contains attributes related to [clusters running on Azure](https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/clusters#--azureattributes).
 
-Here is the example of shared autoscaling cluster with some of AWS options set:
+Here is the example of shared autoscaling cluster with some of Azure options set:
 
 ```hcl
 data "databricks_spark_version" "latest" {}
@@ -351,10 +351,35 @@ The following options are [available](https://docs.microsoft.com/en-us/azure/dat
 
 `gcp_attributes` optional configuration block contains attributes related to [clusters running on GCP](https://docs.gcp.databricks.com/dev-tools/api/latest/clusters.html#clustergcpattributes).
 
+Here is the example of shared autoscaling cluster with some of GCP options set:
+
+```hcl
+resource "databricks_cluster" "this" {
+  cluster_name            = "Shared Autoscaling"
+  spark_version           = data.databricks_spark_version.latest.id
+  node_type_id            = data.databricks_node_type.smallest.id
+  autotermination_minutes = 20
+  autoscale {
+    min_workers = 1
+    max_workers = 50
+  }
+  gcp_attributes {
+    availability = "PREEMPTIBLE_WITH_FALLBACK_GCP"
+    zone_id      = "AUTO"
+  }
+}
+```
+
 The following options are available:
 
-* `use_preemptible_executors` - (Optional, bool) if we should use preemptible executors ([GCP documentation](https://cloud.google.com/compute/docs/instances/preemptible))
+* `use_preemptible_executors` - (Optional, bool) if we should use preemptible executors ([GCP documentation](https://cloud.google.com/compute/docs/instances/preemptible)). *Warning: this field is deprecated in favor of `availability`, and will be removed soon.*
 * `google_service_account` - (Optional, string) Google Service Account email address that the cluster uses to authenticate with Google Identity. This field is used for authentication with the GCS and BigQuery data sources.
+* `availability` - (Optional) Availability type used for all nodes. Valid values are `PREEMPTIBLE_GCP`, `PREEMPTIBLE_WITH_FALLBACK_GCP` and `ON_DEMAND_GCP`, default: `ON_DEMAND_GCP`.
+* `boot_disk_size` (optional, int) Boot disk size in GB
+* `zone_id` (optional)  Identifier for the availability zone in which the cluster resides. This can be one of the following:
+  * `HA` (default): High availability, spread nodes across availability zones for a Databricks deployment region.
+  * `AUTO`: Databricks picks an availability zone to schedule the cluster on.
+  * name of a GCP availability zone: pick one of the available zones from the [list of available availability zones](https://cloud.google.com/compute/docs/regions-zones#available).
 
 ## docker_image
 

docs/resources/instance_pool.md

@@ -61,9 +61,17 @@ The following options are [available](https://docs.databricks.com/dev-tools/api/
 
 The following options are [available](https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/clusters#--azureattributes):
 
-* `availability` - (Optional) Availability type used for all subsequent nodes past the `first_on_demand` ones. Valid values are `SPOT_AZURE` and `ON_DEMAND_AZURE`.
+* `availability` - (Optional) Availability type used for all nodes. Valid values are `SPOT_AZURE` and `ON_DEMAND_AZURE`.
 * `spot_bid_max_price` - (Optional) The max price for Azure spot instances.  Use `-1` to specify lowest price.
 
+## gcp_attributes Configuration Block
+
+`gcp_attributes` optional configuration block contains attributes related to [instance pools on GCP](https://docs.gcp.databricks.com/dev-tools/api/latest/instance-pools.html#clusterinstancepoolgcpattributes).
+
+The following options are [available](https://docs.gcp.databricks.com/dev-tools/api/latest/clusters.html#gcpavailability):
+
+* `availability` - (Optional) Availability type used for all nodes. Valid values are `PREEMPTIBLE_GCP`, `PREEMPTIBLE_WITH_FALLBACK_GCP` and `ON_DEMAND_GCP`, default: `ON_DEMAND_GCP`.
+
 
 ### disk_spec Configuration Block
 

pools/resource_instance_pool.go

@@ -24,6 +24,12 @@ type InstancePoolAzureAttributes struct {
 	SpotBidMaxPrice float64               `json:"spot_bid_max_price,omitempty" tf:"force_new"`
 }
 
+// InstancePoolGcpAttributes contains aws attributes for GCP Databricks deployments for instance pools
+// https://docs.gcp.databricks.com/dev-tools/api/latest/instance-pools.html#instancepoolgcpattributes
+type InstancePoolGcpAttributes struct {
+	Availability clusters.Availability `json:"availability,omitempty" tf:"force_new"`
+}
+
 // InstancePoolDiskType contains disk type information for each of the different cloud service providers
 type InstancePoolDiskType struct {
 	AzureDiskVolumeType string `json:"azure_disk_volume_type,omitempty" tf:"force_new"`
@@ -46,6 +52,7 @@ type InstancePool struct {
 	IdleInstanceAutoTerminationMinutes int32                        `json:"idle_instance_autotermination_minutes"`
 	AwsAttributes                      *InstancePoolAwsAttributes   `json:"aws_attributes,omitempty" tf:"force_new,suppress_diff"`
 	AzureAttributes                    *InstancePoolAzureAttributes `json:"azure_attributes,omitempty" tf:"force_new,suppress_diff"`
+	GcpAttributes                      *InstancePoolGcpAttributes   `json:"gcp_attributes,omitempty" tf:"force_new,suppress_diff"`
 	NodeTypeID                         string                       `json:"node_type_id" tf:"force_new"`
 	CustomTags                         map[string]string            `json:"custom_tags,omitempty" tf:"force_new"`
 	EnableElasticDisk                  bool                         `json:"enable_elastic_disk,omitempty" tf:"force_new,suppress_diff"`
@@ -70,6 +77,7 @@ type InstancePoolAndStats struct {
 	MaxCapacity                        int32                        `json:"max_capacity,omitempty"`
 	AwsAttributes                      *InstancePoolAwsAttributes   `json:"aws_attributes,omitempty"`
 	AzureAttributes                    *InstancePoolAzureAttributes `json:"azure_attributes,omitempty"`
+	GcpAttributes                      *InstancePoolGcpAttributes   `json:"gcp_attributes,omitempty"`
 	NodeTypeID                         string                       `json:"node_type_id"`
 	DefaultTags                        map[string]string            `json:"default_tags,omitempty" tf:"computed"`
 	CustomTags                         map[string]string            `json:"custom_tags,omitempty"`
@@ -135,8 +143,9 @@ func (a InstancePoolsAPI) Delete(instancePoolID string) error {
 func ResourceInstancePool() *schema.Resource {
 	s := common.StructToSchema(InstancePool{}, func(s map[string]*schema.Schema) map[string]*schema.Schema {
 		s["enable_elastic_disk"].Default = true
-		s["aws_attributes"].ConflictsWith = []string{"azure_attributes"}
-		s["azure_attributes"].ConflictsWith = []string{"aws_attributes"}
+		s["aws_attributes"].ConflictsWith = []string{"azure_attributes", "gcp_attributes"}
+		s["azure_attributes"].ConflictsWith = []string{"aws_attributes", "gcp_attributes"}
+		s["gcp_attributes"].ConflictsWith = []string{"azure_attributes", "aws_attributes"}
 		if v, err := common.SchemaPath(s, "aws_attributes", "availability"); err == nil {
 			v.Default = clusters.AwsAvailabilitySpot
 			v.ValidateFunc = validation.StringInSlice([]string{
@@ -154,6 +163,14 @@ func ResourceInstancePool() *schema.Resource {
 				clusters.AzureAvailabilityOnDemand,
 			}, false)
 		}
+		if v, err := common.SchemaPath(s, "gcp_attributes", "availability"); err == nil {
+			v.Default = clusters.GcpAvailabilityOnDemand
+			v.ValidateFunc = validation.StringInSlice([]string{
+				clusters.GcpAvailabilityOnDemand,
+				clusters.GcpAvailabilityPreemptible,
+				clusters.GcpAvailabilityPreemptibleWithFallback,
+			}, false)
+		}
 		if v, err := common.SchemaPath(s, "disk_spec", "disk_type", "azure_disk_volume_type"); err == nil {
 			// nolint
 			v.ValidateFunc = validation.StringInSlice([]string{