google_bigquery_routine

A user-defined function or a stored procedure that belongs to a Dataset

To get more information about Routine, see:

Open in Cloud Shell

Example Usage - Bigquery Routine Basic

resource "google_bigquery_dataset" "test" {
    dataset_id = "dataset_id"
}

resource "google_bigquery_routine" "sproc" {
  dataset_id = google_bigquery_dataset.test.dataset_id
  routine_id     = "routine_id"
  routine_type = "PROCEDURE"
  language = "SQL"
  definition_body = "CREATE FUNCTION Add(x FLOAT64, y FLOAT64) RETURNS FLOAT64 AS (x + y);"
}
Open in Cloud Shell

Example Usage - Bigquery Routine Json

resource "google_bigquery_dataset" "test" {
    dataset_id = "dataset_id"
}

resource "google_bigquery_routine" "sproc" {
  dataset_id = google_bigquery_dataset.test.dataset_id
  routine_id     = "routine_id"
  routine_type = "SCALAR_FUNCTION"
  language = "JAVASCRIPT"
  definition_body = "CREATE FUNCTION multiplyInputs return x*y;"
  arguments {
    name = "x"
    data_type = "{\"typeKind\" :  \"FLOAT64\"}"
  } 
  arguments {
    name = "y"
    data_type = "{\"typeKind\" :  \"FLOAT64\"}"
  }

  return_type = "{\"typeKind\" :  \"FLOAT64\"}"
}
Open in Cloud Shell

Example Usage - Bigquery Routine Tvf

resource "google_bigquery_dataset" "test" {
    dataset_id = "dataset_id"
}

resource "google_bigquery_routine" "sproc" {
  dataset_id      = google_bigquery_dataset.test.dataset_id
  routine_id      = "routine_id"
  routine_type    = "TABLE_VALUED_FUNCTION"
  language        = "SQL"
  definition_body = <<-EOS
    SELECT 1 + value AS value
  EOS
  arguments {
    name          = "value"
    argument_kind = "FIXED_TYPE"
    data_type     = jsonencode({ "typeKind" : "INT64" })
  }
  return_table_type = jsonencode({"columns" : [
    { "name" : "value", "type" : { "typeKind" : "INT64" } },
  ] })
}
Open in Cloud Shell

Example Usage - Bigquery Routine Pyspark

resource "google_bigquery_dataset" "test" {
  dataset_id = "dataset_id"
}

resource "google_bigquery_connection" "test" {
  connection_id = "connection_id"
  location      = "US"
  spark { }
}

resource "google_bigquery_routine" "pyspark" {
  dataset_id      = google_bigquery_dataset.test.dataset_id
  routine_id      = "routine_id"
  routine_type    = "PROCEDURE"
  language        = "PYTHON"
  definition_body = <<-EOS
    from pyspark.sql import SparkSession

    spark = SparkSession.builder.appName("spark-bigquery-demo").getOrCreate()

    # Load data from BigQuery.
    words = spark.read.format("bigquery") \
      .option("table", "bigquery-public-data:samples.shakespeare") \
      .load()
    words.createOrReplaceTempView("words")

    # Perform word count.
    word_count = words.select('word', 'word_count').groupBy('word').sum('word_count').withColumnRenamed("sum(word_count)", "sum_word_count")
    word_count.show()
    word_count.printSchema()

    # Saving the data to BigQuery
    word_count.write.format("bigquery") \
      .option("writeMethod", "direct") \
      .save("wordcount_dataset.wordcount_output")
  EOS
  spark_options {
    connection          = google_bigquery_connection.test.name
    runtime_version     = "2.1"
  }
}
Open in Cloud Shell

Example Usage - Bigquery Routine Pyspark Mainfile

resource "google_bigquery_dataset" "test" {
  dataset_id = "dataset_id"
}

resource "google_bigquery_connection" "test" {
  connection_id = "connection_id"
  location      = "US"
  spark { }
}

resource "google_bigquery_routine" "pyspark_mainfile" {
  dataset_id      = google_bigquery_dataset.test.dataset_id
  routine_id      = "routine_id"
  routine_type    = "PROCEDURE"
  language        = "PYTHON"
  definition_body = ""
  spark_options {
    connection      = google_bigquery_connection.test.name
    runtime_version = "2.1"
    main_file_uri   = "gs://test-bucket/main.py"
    py_file_uris    = ["gs://test-bucket/lib.py"]
    file_uris       = ["gs://test-bucket/distribute_in_executor.json"]
    archive_uris    = ["gs://test-bucket/distribute_in_executor.tar.gz"]
  }
}
Open in Cloud Shell

Example Usage - Bigquery Routine Spark Jar

resource "google_bigquery_dataset" "test" {
  dataset_id = "dataset_id"
}

resource "google_bigquery_connection" "test" {
  connection_id = "connection_id"
  location      = "US"
  spark { }
}

resource "google_bigquery_routine" "spark_jar" {
  dataset_id      = google_bigquery_dataset.test.dataset_id
  routine_id      = "routine_id"
  routine_type    = "PROCEDURE"
  language        = "SCALA"
  definition_body = ""
  spark_options {
    connection      = google_bigquery_connection.test.name
    runtime_version = "2.1"
    container_image = "gcr.io/my-project-id/my-spark-image:latest"
    main_class      = "com.google.test.jar.MainClass"
    jar_uris        = [ "gs://test-bucket/uberjar_spark_spark3.jar" ]
    properties      = {
      "spark.dataproc.scaling.version" : "2",
      "spark.reducer.fetchMigratedShuffle.enabled" : "true",
    }
  }
}
Open in Cloud Shell

Example Usage - Bigquery Routine Data Governance Type

resource "google_bigquery_dataset" "test" {
    dataset_id = "tf_test_dataset_id%{random_suffix}"
}

resource "google_bigquery_routine" "custom_masking_routine" {
    dataset_id = google_bigquery_dataset.test.dataset_id
    routine_id     = "custom_masking_routine"
    routine_type = "SCALAR_FUNCTION"
    language = "SQL"
    data_governance_type = "DATA_MASKING"
    definition_body = "SAFE.REGEXP_REPLACE(ssn, '[0-9]', 'X')"
    arguments {
      name = "ssn"
      data_type = "{\"typeKind\" :  \"STRING\"}"
    } 
    return_type = "{\"typeKind\" :  \"STRING\"}"
  }

Example Usage - Bigquery Routine Remote Function

resource "google_bigquery_dataset" "test" {
  dataset_id = "dataset_id"
}

resource "google_bigquery_connection" "test" {
  connection_id = "connection_id"
  location      = "US"
  cloud_resource { }
}

resource "google_bigquery_routine" "remote_function" {
  dataset_id = google_bigquery_dataset.test.dataset_id
  routine_id = "routine_id"
  routine_type = "SCALAR_FUNCTION"
  definition_body = ""

  return_type = "{\"typeKind\" :  \"STRING\"}"

  remote_function_options {
    endpoint = "https://us-east1-my_gcf_project.cloudfunctions.net/remote_add"
    connection = google_bigquery_connection.test.name
    max_batching_rows = "10"
    user_defined_context = {
      "z": "1.5",
    }
  }
}

Argument Reference

The following arguments are supported:


The arguments block supports:

The spark_options block supports:

The remote_function_options block supports:

Attributes Reference

In addition to the arguments listed above, the following computed attributes are exported:

Timeouts

This resource provides the following Timeouts configuration options:

Import

Routine can be imported using any of these accepted formats:

In Terraform v1.5.0 and later, use an import block to import Routine using one of the formats above. For example:

import {
  id = "projects/{{project}}/datasets/{{dataset_id}}/routines/{{routine_id}}"
  to = google_bigquery_routine.default
}

When using the terraform import command, Routine can be imported using one of the formats above. For example:

$ terraform import google_bigquery_routine.default projects/{{project}}/datasets/{{dataset_id}}/routines/{{routine_id}}
$ terraform import google_bigquery_routine.default {{project}}/{{dataset_id}}/{{routine_id}}
$ terraform import google_bigquery_routine.default {{dataset_id}}/{{routine_id}}

User Project Overrides

This resource supports User Project Overrides.