google_datastream_stream

A resource representing streaming data from a source to a destination.

To get more information about Stream, see:

Open in Cloud Shell

Example Usage - Datastream Stream Full

data "google_project" "project" {
}

resource "google_sql_database_instance" "instance" {
    name             = "my-instance"
    database_version = "MYSQL_8_0"
    region           = "us-central1"
    settings {
        tier = "db-f1-micro"
        backup_configuration {
            enabled            = true
            binary_log_enabled = true
        }

        ip_configuration {

            // Datastream IPs will vary by region.
            authorized_networks {
                value = "34.71.242.81"
            }

            authorized_networks {
                value = "34.72.28.29"
            }

            authorized_networks {
                value = "34.67.6.157"
            }

            authorized_networks {
                value = "34.67.234.134"
            }

            authorized_networks {
                value = "34.72.239.218"
            }
        }
    }

    deletion_protection  = true
}

resource "google_sql_database" "db" {
    instance = google_sql_database_instance.instance.name
    name     = "db"
}

resource "random_password" "pwd" {
    length = 16
    special = false
}

resource "google_sql_user" "user" {
    name     = "user"
    instance = google_sql_database_instance.instance.name
    host     = "%"
    password = random_password.pwd.result
}

resource "google_datastream_connection_profile" "source_connection_profile" {
    display_name          = "Source connection profile"
    location              = "us-central1"
    connection_profile_id = "source-profile"

    mysql_profile {
        hostname = google_sql_database_instance.instance.public_ip_address
        username = google_sql_user.user.name
        password = google_sql_user.user.password
    }
}

resource "google_storage_bucket" "bucket" {
  name                        = "my-bucket"
  location                    = "US"
  uniform_bucket_level_access = true
}

resource "google_storage_bucket_iam_member" "viewer" {
    bucket = google_storage_bucket.bucket.name
    role   = "roles/storage.objectViewer"
    member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com"
}

resource "google_storage_bucket_iam_member" "creator" {
    bucket = google_storage_bucket.bucket.name
    role   = "roles/storage.objectCreator"
    member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com"
}

resource "google_storage_bucket_iam_member" "reader" {
    bucket = google_storage_bucket.bucket.name
    role   = "roles/storage.legacyBucketReader"
    member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com"
}

resource "google_kms_crypto_key_iam_member" "key_user" {
    crypto_key_id = "kms-name"
    role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
    member        = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-datastream.iam.gserviceaccount.com"
}

resource "google_datastream_connection_profile" "destination_connection_profile" {
    display_name          = "Connection profile"
    location              = "us-central1"
    connection_profile_id = "destination-profile"

    gcs_profile {
        bucket    = google_storage_bucket.bucket.name
        root_path = "/path"
    }
}

resource "google_datastream_stream" "default" {
    depends_on = [
        google_kms_crypto_key_iam_member.key_user
    ]
    stream_id = "my-stream"
    desired_state = "NOT_STARTED"
    location = "us-central1"
    display_name = "my stream"
    labels = {
        key = "value"
    }
    source_config {
        source_connection_profile = google_datastream_connection_profile.source_connection_profile.id
        mysql_source_config {
            include_objects {
                mysql_databases {
                    database = "my-database"
                    mysql_tables {
                        table = "includedTable"
                        mysql_columns {
                            column = "includedColumn"
                            data_type = "VARCHAR"
                            collation = "utf8mb4"
                            primary_key = false
                            nullable = false
                            ordinal_position = 0
                        }
                    }
                    mysql_tables {
                        table = "includedTable_2"
                    }
                }
            }
            exclude_objects {
                mysql_databases {
                    database = "my-database"
                    mysql_tables {
                        table = "excludedTable"
                        mysql_columns {
                            column = "excludedColumn"
                            data_type = "VARCHAR"
                            collation = "utf8mb4"
                            primary_key = false
                            nullable = false
                            ordinal_position = 0
                        }
                    }
                }
            }
            max_concurrent_cdc_tasks = 5
        }
    }
    destination_config {
        destination_connection_profile = google_datastream_connection_profile.destination_connection_profile.id
        gcs_destination_config {
            path = "mydata"
            file_rotation_mb = 200
            file_rotation_interval = "60s"
            json_file_format {
                schema_file_format = "NO_SCHEMA_FILE"
                compression = "GZIP"
            }
        }
    }

    backfill_all {
        mysql_excluded_objects {
            mysql_databases {
                database = "my-database"
                mysql_tables {
                    table = "excludedTable"
                    mysql_columns {
                        column = "excludedColumn"
                        data_type = "VARCHAR"
                        collation = "utf8mb4"
                        primary_key = false
                        nullable = false
                        ordinal_position = 0
                    }
                }
            }
        }
    }

    customer_managed_encryption_key = "kms-name"
}
## Example Usage - Datastream Stream Postgresql
resource "google_datastream_connection_profile" "source" {
    display_name          = "Postgresql Source"
    location              = "us-central1"
    connection_profile_id = "source-profile"

    postgresql_profile {
        hostname = "hostname"
        port     = 3306
        username = "user"
        password = "pass"
        database = "postgres"
    }
}

resource "google_datastream_connection_profile" "destination" {
    display_name          = "BigQuery Destination"
    location              = "us-central1"
    connection_profile_id = "destination-profile"

    bigquery_profile {}
}

resource "google_datastream_stream" "default"  {
    display_name = "Postgres to BigQuery"
    location     = "us-central1"
    stream_id    = "my-stream"
    desired_state = "RUNNING"

    source_config {
        source_connection_profile = google_datastream_connection_profile.source.id
        postgresql_source_config {
            max_concurrent_backfill_tasks = 12
            publication      = "publication"
            replication_slot = "replication_slot"
            include_objects {
                postgresql_schemas {
                    schema = "schema"
                    postgresql_tables {
                        table = "table"
                        postgresql_columns {
                            column = "column"
                        }
                    }
                }
            }
            exclude_objects {
                postgresql_schemas {
                    schema = "schema"
                    postgresql_tables {
                        table = "table"
                        postgresql_columns {
                            column = "column"
                        }
                    }
                }
            }
        }
    }

    destination_config {
        destination_connection_profile = google_datastream_connection_profile.destination.id
        bigquery_destination_config {
            data_freshness = "900s"
            source_hierarchy_datasets {
                dataset_template {
                   location = "us-central1"
                }
            }
        }
    }

    backfill_all {
        postgresql_excluded_objects {
            postgresql_schemas {
                schema = "schema"
                postgresql_tables {
                    table = "table"
                    postgresql_columns {
                        column = "column"
                    }
                }
            }
        }
    }
}
## Example Usage - Datastream Stream Oracle
resource "google_datastream_connection_profile" "source" {
    display_name          = "Oracle Source"
    location              = "us-central1"
    connection_profile_id = "source-profile"

    oracle_profile {
        hostname = "hostname"
        port     = 1521
        username = "user"
        password = "pass"
        database_service = "ORCL"
    }
}

resource "google_datastream_connection_profile" "destination" {
    display_name          = "BigQuery Destination"
    location              = "us-central1"
    connection_profile_id = "destination-profile"

    bigquery_profile {}
}

resource "google_datastream_stream" "stream5" {
    display_name = "Oracle to BigQuery"
    location     = "us-central1"
    stream_id    = "my-stream"
    desired_state = "RUNNING"

    source_config {
        source_connection_profile = google_datastream_connection_profile.source.id
        oracle_source_config {
            max_concurrent_cdc_tasks = 8
            max_concurrent_backfill_tasks = 12
            include_objects {
                oracle_schemas {
                    schema = "schema"
                    oracle_tables {
                        table = "table"
                        oracle_columns {
                            column = "column"
                        }
                    }
                }
            }
            exclude_objects {
                oracle_schemas {
                    schema = "schema"
                    oracle_tables {
                        table = "table"
                        oracle_columns {
                            column = "column"
                        }
                    }
                }
            }
            drop_large_objects {}
            }
    }

    destination_config {
        destination_connection_profile = google_datastream_connection_profile.destination.id
        bigquery_destination_config {
            data_freshness = "900s"
            source_hierarchy_datasets {
                dataset_template {
                    location = "us-central1"
                }
            }
        }
    }

    backfill_all {
        oracle_excluded_objects {
            oracle_schemas {
                schema = "schema"
                oracle_tables {
                    table = "table"
                    oracle_columns {
                        column = "column"
                    }
                }
            }
        }
    }
}
Open in Cloud Shell

Example Usage - Datastream Stream Postgresql Bigquery Dataset Id

resource "google_bigquery_dataset" "postgres" {
  dataset_id    = "postgres"
  friendly_name = "postgres"
  description   = "Database of postgres"
  location      = "us-central1"
}

resource "google_datastream_stream" "default" {
  display_name  = "postgres to bigQuery"
  location      = "us-central1"
  stream_id     = "postgres-bigquery"

   source_config {
    source_connection_profile = google_datastream_connection_profile.source_connection_profile.id
    mysql_source_config {}
  }

  destination_config {
    destination_connection_profile = google_datastream_connection_profile.destination_connection_profile2.id
    bigquery_destination_config {
      data_freshness = "900s"
      single_target_dataset {
        dataset_id = google_bigquery_dataset.postgres.id
      }
    }
  }

  backfill_all {
  }

}

resource "google_datastream_connection_profile" "destination_connection_profile2" {
    display_name          = "Connection profile"
    location              = "us-central1"
    connection_profile_id = "dest-profile"
    bigquery_profile {}
}

resource "google_sql_database_instance" "instance" {
    name             = "instance-name"
    database_version = "MYSQL_8_0"
    region           = "us-central1"
    settings {
        tier = "db-f1-micro"
        backup_configuration {
            enabled            = true
            binary_log_enabled = true
        }

        ip_configuration {
            // Datastream IPs will vary by region.
            authorized_networks {
                value = "34.71.242.81"
            }

            authorized_networks {
                value = "34.72.28.29"
            }

            authorized_networks {
                value = "34.67.6.157"
            }

            authorized_networks {
                value = "34.67.234.134"
            }

            authorized_networks {
                value = "34.72.239.218"
            }
        }
    }

    deletion_protection  = false
}

resource "google_sql_database" "db" {
    instance = google_sql_database_instance.instance.name
    name     = "db"
}

resource "random_password" "pwd" {
    length = 16
    special = false
}

resource "google_sql_user" "user" {
    name     = "my-user"
    instance = google_sql_database_instance.instance.name
    host     = "%"
    password = random_password.pwd.result
}

resource "google_datastream_connection_profile" "source_connection_profile" {
    display_name          = "Source connection profile"
    location              = "us-central1"
    connection_profile_id = "source-profile"

    mysql_profile {
        hostname = google_sql_database_instance.instance.public_ip_address
        username = google_sql_user.user.name
        password = google_sql_user.user.password
    }
}
Open in Cloud Shell

Example Usage - Datastream Stream Bigquery

data "google_project" "project" {
}

resource "google_sql_database_instance" "instance" {
    name             = "my-instance"
    database_version = "MYSQL_8_0"
    region           = "us-central1"
    settings {
        tier = "db-f1-micro"
        backup_configuration {
            enabled            = true
            binary_log_enabled = true
        }

        ip_configuration {

            // Datastream IPs will vary by region.
            authorized_networks {
                value = "34.71.242.81"
            }

            authorized_networks {
                value = "34.72.28.29"
            }

            authorized_networks {
                value = "34.67.6.157"
            }

            authorized_networks {
                value = "34.67.234.134"
            }

            authorized_networks {
                value = "34.72.239.218"
            }
        }
    }

    deletion_protection  = true
}

resource "google_sql_database" "db" {
    instance = google_sql_database_instance.instance.name
    name     = "db"
}

resource "random_password" "pwd" {
    length = 16
    special = false
}

resource "google_sql_user" "user" {
    name     = "user"
    instance = google_sql_database_instance.instance.name
    host     = "%"
    password = random_password.pwd.result
}

resource "google_datastream_connection_profile" "source_connection_profile" {
    display_name          = "Source connection profile"
    location              = "us-central1"
    connection_profile_id = "source-profile"

    mysql_profile {
        hostname = google_sql_database_instance.instance.public_ip_address
        username = google_sql_user.user.name
        password = google_sql_user.user.password
    }
}

data "google_bigquery_default_service_account" "bq_sa" {
}

resource "google_kms_crypto_key_iam_member" "bigquery_key_user" {
  crypto_key_id = "bigquery-kms-name"
  role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
  member        = "serviceAccount:${data.google_bigquery_default_service_account.bq_sa.email}"
}

resource "google_datastream_connection_profile" "destination_connection_profile" {
    display_name          = "Connection profile"
    location              = "us-central1"
    connection_profile_id = "destination-profile"

    bigquery_profile {}
}

resource "google_datastream_stream" "default" {
    depends_on = [
        google_kms_crypto_key_iam_member.bigquery_key_user
    ]
    stream_id = "my-stream"
    location = "us-central1"
    display_name = "my stream"
    source_config {
        source_connection_profile = google_datastream_connection_profile.source_connection_profile.id
        mysql_source_config {}
    }
    destination_config {
        destination_connection_profile = google_datastream_connection_profile.destination_connection_profile.id
        bigquery_destination_config {
            source_hierarchy_datasets {
                dataset_template {
                    location = "us-central1"
                    kms_key_name = "bigquery-kms-name"
                }
            }
        }
    }

    backfill_none {
    }
}

Argument Reference

The following arguments are supported:

The source_config block supports:

The mysql_source_config block supports:

The include_objects block supports:

The mysql_databases block supports:

The mysql_tables block supports:

The mysql_columns block supports:

The exclude_objects block supports:

The mysql_databases block supports:

The mysql_tables block supports:

The mysql_columns block supports:

The oracle_source_config block supports:

The include_objects block supports:

The oracle_schemas block supports:

The oracle_tables block supports:

The oracle_columns block supports:

The exclude_objects block supports:

The oracle_schemas block supports:

The oracle_tables block supports:

The oracle_columns block supports:

The postgresql_source_config block supports:

The include_objects block supports:

The postgresql_schemas block supports:

The postgresql_tables block supports:

The postgresql_columns block supports:

The exclude_objects block supports:

The postgresql_schemas block supports:

The postgresql_tables block supports:

The postgresql_columns block supports:

The destination_config block supports:

The gcs_destination_config block supports:

The json_file_format block supports:

The bigquery_destination_config block supports:

The single_target_dataset block supports:

The source_hierarchy_datasets block supports:

The dataset_template block supports:


The backfill_all block supports:

The mysql_excluded_objects block supports:

The mysql_databases block supports:

The mysql_tables block supports:

The mysql_columns block supports:

The postgresql_excluded_objects block supports:

The postgresql_schemas block supports:

The postgresql_tables block supports:

The postgresql_columns block supports:

The oracle_excluded_objects block supports:

The oracle_schemas block supports:

The oracle_tables block supports:

The oracle_columns block supports:

Attributes Reference

In addition to the arguments listed above, the following computed attributes are exported:

Timeouts

This resource provides the following Timeouts configuration options:

Import

Stream can be imported using any of these accepted formats:

In Terraform v1.5.0 and later, use an import block to import Stream using one of the formats above. For example:

import {
  id = "projects/{{project}}/locations/{{location}}/streams/{{stream_id}}"
  to = google_datastream_stream.default
}

When using the terraform import command, Stream can be imported using one of the formats above. For example:

$ terraform import google_datastream_stream.default projects/{{project}}/locations/{{location}}/streams/{{stream_id}}
$ terraform import google_datastream_stream.default {{project}}/{{location}}/{{stream_id}}
$ terraform import google_datastream_stream.default {{location}}/{{stream_id}}

User Project Overrides

This resource supports User Project Overrides.