azurerm_data_factory_data_flow

Manages a Data Flow inside an Azure Data Factory.

Example Usage

resource "azurerm_resource_group" "example" {
  name     = "example-resources"
  location = "West Europe"
}

resource "azurerm_storage_account" "example" {
  name                     = "example"
  location                 = azurerm_resource_group.example.location
  resource_group_name      = azurerm_resource_group.example.name
  account_tier             = "Standard"
  account_replication_type = "LRS"
}

resource "azurerm_data_factory" "example" {
  name                = "example"
  location            = azurerm_resource_group.example.location
  resource_group_name = azurerm_resource_group.example.name
}

resource "azurerm_data_factory_linked_custom_service" "example" {
  name                 = "linked_service"
  data_factory_id      = azurerm_data_factory.example.id
  type                 = "AzureBlobStorage"
  type_properties_json = <<JSON
{
  "connectionString": "${azurerm_storage_account.example.primary_connection_string}"
}
JSON
}

resource "azurerm_data_factory_dataset_json" "example1" {
  name                = "dataset1"
  data_factory_id     = azurerm_data_factory.example.id
  linked_service_name = azurerm_data_factory_linked_custom_service.example.name

  azure_blob_storage_location {
    container = "container"
    path      = "foo/bar/"
    filename  = "foo.txt"
  }

  encoding = "UTF-8"
}

resource "azurerm_data_factory_dataset_json" "example2" {
  name                = "dataset2"
  data_factory_id     = azurerm_data_factory.example.id
  linked_service_name = azurerm_data_factory_linked_custom_service.example.name

  azure_blob_storage_location {
    container = "container"
    path      = "foo/bar/"
    filename  = "bar.txt"
  }

  encoding = "UTF-8"
}

resource "azurerm_data_factory_data_flow" "example" {
  name            = "example"
  data_factory_id = azurerm_data_factory.example.id

  source {
    name = "source1"

    flowlet {
      name = azurerm_data_factory_flowlet_data_flow.example1.name
      parameters = {
        "Key1" = "value1"
      }
    }

    dataset {
      name = azurerm_data_factory_dataset_json.example1.name
    }
  }

  sink {
    name = "sink1"

    flowlet {
      name = azurerm_data_factory_flowlet_data_flow.example2.name
      parameters = {
        "Key1" = "value1"
      }
    }

    dataset {
      name = azurerm_data_factory_dataset_json.example2.name
    }
  }

  script = <<EOT
source(
  allowSchemaDrift: true, 
  validateSchema: false, 
  limit: 100, 
  ignoreNoFilesFound: false, 
  documentForm: 'documentPerLine') ~> source1 
source1 sink(
  allowSchemaDrift: true, 
  validateSchema: false, 
  skipDuplicateMapInputs: true, 
  skipDuplicateMapOutputs: true) ~> sink1
EOT
}

resource "azurerm_data_factory_flowlet_data_flow" "example1" {
  name            = "example"
  data_factory_id = azurerm_data_factory.example.id

  source {
    name = "source1"

    linked_service {
      name = azurerm_data_factory_linked_custom_service.example.name
    }
  }

  sink {
    name = "sink1"

    linked_service {
      name = azurerm_data_factory_linked_custom_service.example.name
    }
  }

  script = <<EOT
source(
  allowSchemaDrift: true, 
  validateSchema: false, 
  limit: 100, 
  ignoreNoFilesFound: false, 
  documentForm: 'documentPerLine') ~> source1 
source1 sink(
  allowSchemaDrift: true, 
  validateSchema: false, 
  skipDuplicateMapInputs: true, 
  skipDuplicateMapOutputs: true) ~> sink1
EOT
}

resource "azurerm_data_factory_flowlet_data_flow" "example2" {
  name            = "example"
  data_factory_id = azurerm_data_factory.example.id

  source {
    name = "source1"

    linked_service {
      name = azurerm_data_factory_linked_custom_service.example.name
    }
  }

  sink {
    name = "sink1"

    linked_service {
      name = azurerm_data_factory_linked_custom_service.example.name
    }
  }

  script = <<EOT
source(
  allowSchemaDrift: true, 
  validateSchema: false, 
  limit: 100, 
  ignoreNoFilesFound: false, 
  documentForm: 'documentPerLine') ~> source1 
source1 sink(
  allowSchemaDrift: true, 
  validateSchema: false, 
  skipDuplicateMapInputs: true, 
  skipDuplicateMapOutputs: true) ~> sink1
EOT
}

Argument Reference

The following arguments are supported:


A source block supports the following:


A sink block supports the following:


A dataset block supports the following:


A flowlet block supports the following:


A linked_service block supports the following:


A rejected_linked_service block supports the following:


A schema_linked_service block supports the following:


A transformation block supports the following:

Attributes Reference

In addition to the Arguments listed above - the following Attributes are exported:

Timeouts

The timeouts block allows you to specify timeouts for certain actions:

Import

Data Factory Data Flow can be imported using the resource id, e.g.

terraform import azurerm_data_factory_data_flow.example /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/example/providers/Microsoft.DataFactory/factories/example/dataflows/example