Add adf and fix AF to use id

Author: helayoty

infra/modules/providers/azure/data-factory/README.md

@@ -17,15 +17,13 @@ An instance of the `data-factory` module deploys the _**Data Factory**_ in order
 - Ability to provision a single Data Factory instance
 - Ability to provision a configurable Pipeline
 - Ability to configure Trigger
-- Ability to configure SQL server Dataset
-- Ability to configure SQL server Linked Service
+
 
 ## Out Of Scope
 
 The following are not support in the time being
 
 - Creating Multiple pipelines
-- Only SQL server Dataset/Linked Service are implemented.
 
 ## Definition
 
@@ -35,8 +33,6 @@ Terraform resources used to define the `data-factory` module include the followi
 - [azurerm_data_factory_integration_runtime_managed](https://www.terraform.io/docs/providers/azurerm/r/data_factory_integration_runtime_managed.html)
 - [azurerm_data_factory_pipeline](https://www.terraform.io/docs/providers/azurerm/r/data_factory_pipeline.html)
 - [azurerm_data_factory_trigger_schedule](https://www.terraform.io/docs/providers/azurerm/r/data_factory_trigger_schedule.html)
-- [azurerm_data_factory_dataset_sql_server](https://www.terraform.io/docs/providers/azurerm/r/data_factory_dataset_sql_server_table.html)
-- [azurerm_data_factory_linked_service_sql_server](https://www.terraform.io/docs/providers/azurerm/r/data_factory_linked_service_sql_server.html)
 
 ## Usage
 
@@ -60,11 +56,6 @@ module "data_factory" {
   data_factory_trigger_name                 = "adftrigger"
   data_factory_trigger_interval             = 1
   data_factory_trigger_frequency            = "Minute"
-  data_factory_dataset_sql_name             = "adfsqldataset"
-  data_factory_dataset_sql_table_name       = "adfsqldatasettable"
-  data_factory_dataset_sql_folder           = ""
-  data_factory_linked_sql_name              = "adfsqllinked"
-  data_factory_linked_sql_connection_string = "Server=tcp:adfsql..."
 }
 ```
 

infra/modules/providers/azure/data-factory/datasets.tf

@@ -28,16 +28,6 @@ output "trigger_interval" {
   value       = azurerm_data_factory_trigger_schedule.main.interval
 }
 
-output "sql_dataset_id" {
-  description = "The ID of the SQL server dataset created"
-  value       = azurerm_data_factory_dataset_sql_server_table.main.id
-}
-
-output "sql_linked_service_id" {
-  description = "The ID of the SQL server Linked service created"
-  value       = azurerm_data_factory_linked_service_sql_server.main.id
-}
-
 output "adf_identity_principal_id" {
   description = "The ID of the principal(client) in Azure active directory"
   value       = azurerm_data_factory.main.identity[0].principal_id

infra/modules/providers/azure/data-factory/linkedservices.tf

@@ -2,10 +2,7 @@ resource_group_name                       = ""
 data_factory_name                         = ""
 data_factory_runtime_name                 = ""
 data_factory_pipeline_name                = ""
-data_factory_dataset_sql_name             = ""
-data_factory_dataset_sql_table_name       = ""
-data_factory_linked_sql_name              = ""
-data_factory_linked_sql_connection_string = ""
+data_factory_trigger_name = ""
 vnet_integration = {
   vnet_id     = ""
   subnet_name = ""

infra/modules/providers/azure/data-factory/output.tf

@@ -25,16 +25,6 @@ func TestDataFactory(t *testing.T) {
 				"data_factory_name",
 				"pipeline_name",
 			),
-			VerifyCreatedDataset(subscription,
-				"resource_group_name",
-				"data_factory_name",
-				"sql_dataset_id",
-			),
-			VerifyCreatedLinkedService(subscription,
-				"resource_group_name",
-				"data_factory_name",
-				"sql_linked_service_id",
-			),
 		},
 	}
 	integration.RunIntegrationTests(&testFixture)

infra/modules/providers/azure/data-factory/terraform.tfvars.template

@@ -1,13 +1,9 @@
-resource_group_name                       = "adftest"
-data_factory_name                         = "adftest"
-data_factory_runtime_name                 = "adfrttest"
-data_factory_pipeline_name                = "testpipeline"
-data_factory_trigger_name                 = "testtrigger"
-data_factory_dataset_sql_name             = "testsql"
-data_factory_dataset_sql_table_name       = "adfsqltableheba"
-data_factory_linked_sql_name              = "testlinkedsql"
-data_factory_linked_sql_connection_string = "connectionstring"
+resource_group_name                       = ""
+data_factory_name                         = ""
+data_factory_runtime_name                 = ""
+data_factory_pipeline_name                = ""
+data_factory_trigger_name = ""
 vnet_integration = {
-  vnet_id     = "/subscriptions/resourceGroups/providers/Microsoft.Network/virtualNetworks/testvnet"
-  subnet_name = "default"
+  vnet_id     = ""
+  subnet_name = ""
 }

infra/modules/providers/azure/data-factory/tests/integration/data_factory_integration_test.go

@@ -1,24 +1,11 @@
 package unit
 
 import (
-	"encoding/json"
-	"strings"
 	"testing"
-
-	"github.com/gruntwork-io/terratest/modules/random"
 	tests "github.com/microsoft/cobalt/infra/modules/providers/azure/data-factory/tests"
 	"github.com/microsoft/terratest-abstraction/unit"
 )
 
-// helper function to parse blocks of JSON into a generic Go map
-func asMap(t *testing.T, jsonString string) map[string]interface{} {
-	var theMap map[string]interface{}
-	if err := json.Unmarshal([]byte(jsonString), &theMap); err != nil {
-		t.Fatal(err)
-	}
-	return theMap
-}
-
 func TestTemplate(t *testing.T) {
 
 	expectedDataFactory := map[string]interface{}{
@@ -53,27 +40,16 @@ func TestTemplate(t *testing.T) {
 		"frequency": "Minute",
 	}
 
-	expectedDatasetSQL := map[string]interface{}{
-		"name": "testsql",
-	}
-
-	expectedLinkedSQL := map[string]interface{}{
-		"name":              "testlinkedsql",
-		"connection_string": "connectionstring",
-	}
-
 	testFixture := unit.UnitTestFixture{
 		GoTest:                t,
 		TfOptions:             tests.DataFactoryTFOptions,
 		PlanAssertions:        nil,
-		ExpectedResourceCount: 6,
+		ExpectedResourceCount: 4,
 		ExpectedResourceAttributeValues: unit.ResourceDescription{
 			"azurerm_data_factory.main":                             expectedDataFactory,
 			"azurerm_data_factory_integration_runtime_managed.main": expectedDFIntRunTime,
 			"azurerm_data_factory_pipeline.main":                    expectedPipeline,
 			"azurerm_data_factory_trigger_schedule.main":            expectedTrigger,
-			"azurerm_data_factory_dataset_sql_server_table.main":    expectedDatasetSQL,
-			"azurerm_data_factory_linked_service_sql_server.main":   expectedLinkedSQL,
 		},
 	}
 

infra/modules/providers/azure/data-factory/tests/test.tfvars

@@ -68,34 +68,4 @@ variable "data_factory_trigger_frequency" {
   description = "The trigger freqency. Valid values include Minute, Hour, Day, Week, Month. Defaults to Minute."
   type        = string
   default     = "Minute"
-}
-
-variable "data_factory_dataset_sql_name" {
-  description = "Specifies the name of the Data Factory Dataset SQL Server Table. Only letters, numbers and '_' are allowed."
-  type        = string
-  default     = ""
-}
-
-variable "data_factory_dataset_sql_table_name" {
-  description = "The table name of the Data Factory Dataset SQL Server Table."
-  type        = string
-  default     = ""
-}
-
-variable "data_factory_dataset_sql_folder" {
-  description = "The folder that this Dataset is in. If not specified, the Dataset will appear at the root level."
-  type        = string
-  default     = ""
-}
-
-variable "data_factory_linked_sql_name" {
-  description = "Specifies the name of the Data Factory Linked Service SQL Server. Changing this forces a new resource to be created."
-  type        = string
-  default     = ""
-}
-
-variable "data_factory_linked_sql_connection_string" {
-  description = "The connection string in which to authenticate with the SQL Server."
-  type        = string
-  default     = ""
 }

infra/modules/providers/azure/data-factory/tests/unit/data_factory_unit_test.go

@@ -98,17 +98,29 @@ module "function_app" {
   source                              = "../../modules/providers/azure/function-app"
   fn_name_prefix                      = local.func_app_name_prefix
   resource_group_name                 = azurerm_resource_group.app_rg.name
-  service_plan_name                   = module.func_app_service_plan.service_plan_name
+  service_plan_id                     = module.func_app_service_plan.app_service_plan_id
   storage_account_resource_group_name = module.sys_storage_account.resource_group_name
   storage_account_name                = module.sys_storage_account.name
   vnet_subnet_id                      = module.network.subnet_ids[0]
   fn_app_settings                     = local.func_app_settings
   fn_app_config                       = var.fn_app_config
 }
 
-module "data-factory" {
-  source              = "../../modules/providers/azure/data-factory"
-  data_factory_name   = local.data_factory_name
-  resource_group_name = azurerm_resource_group.app_rg.name
+
+module "data_factory" {
+  source                           = "../../modules/providers/azure/data-factory"
+  data_factory_name                = local.data_factory_name
+  resource_group_name              = azurerm_resource_group.app_rg.name
+  data_factory_runtime_name        = local.data_factory_runtime_name
+  number_of_nodes                  = var.adf_number_of_nodes
+  max_parallel_executions_per_node = var.adf_max_parallel_executions_per_node
+  vnet_integration = {
+    vnet_id     = module.network.subnet_ids[0]
+    subnet_name = local.subnet_name
+  }
+  data_factory_pipeline_name     = local.data_factory_pipeline_name
+  data_factory_trigger_name      = local.data_factory_trigger_name
+  data_factory_trigger_interval  = var.adf_trigger_interval
+  data_factory_trigger_frequency = var.adf_trigger_frequency
 }
 

infra/modules/providers/azure/data-factory/variables.tf

@@ -66,6 +66,9 @@ locals {
   auth_svc_name_prefix        = "${local.base_name_21}-au"
   smpl_app_postfix            = "app"
   data_factory_name           = "adf-cares-${local.base_name_46}"
+  data_factory_runtime_name   = "adf-rt-cares-${local.base_name_46}"
+  data_factory_pipeline_name  = "adf-pipeline-cares-${local.base_name_83}"
+  data_factory_trigger_name   = "adf-trigger-cares-${local.base_name_83}"
   func_app_name_prefix        = "func-cares-${local.base_name_46}"      // Function app name (Max 60 chars)
   cosmos_account_name         = "cosmos-cares-${local.base_name_21}"    // Cosmos databse account name (max 44 chars)
   cosmos_db_name              = "cosmos-cares-db-${local.base_name_21}" // Cosmos databse account name (max 44 chars)

infra/templates/az-svc-data-integration-mlw/app.tf

@@ -252,6 +252,33 @@ variable "fn_app_config" {
   default = {}
 }
 
+// ---- Azure Data Factory Configuration ----
+
+variable "adf_number_of_nodes" {
+  description = "Number of nodes for the Managed Integration Runtime. Max is 10. Defaults to 1."
+  type        = number
+  default     = 1
+}
+
+variable "adf_max_parallel_executions_per_node" {
+  description = " Defines the maximum parallel executions per node. Defaults to 1. Max is 16."
+  type        = number
+  default     = 1
+}
+
+variable "adf_trigger_interval" {
+  description = "The interval for how often the trigger occurs. This defaults to 1."
+  type        = number
+  default     = 1
+}
+
+variable "adf_trigger_frequency" {
+  description = "The trigger freqency. Valid values include Minute, Hour, Day, Week, Month. Defaults to Minute."
+  type        = string
+  default     = "Minute"
+}
+
+
 // ---- Cosmos Database Configuration ----
 variable "primary_replica_location" {
   description = "The primary replica location for the cosmos database"