{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":[]},"redocly_category":"Integrations","type":"markdown"},"seo":{"title":"Google Bigquery Import Integration CLI","description":"Learn how to import data from Google BigQuery using TD Toolbelt CLI, with step-by-step configuration for authentication, table selection, and incremental imports.","siteUrl":"https://docs.treasure.ai","lang":"en-US","jsonLd":{"@context":"https://schema.org","@graph":[{"@type":"Organization","@id":"https://www.treasure.ai/","name":"Treasure AI","url":"https://www.treasure.ai/","logo":"https://www.treasure.ai/hubfs/assets/images/logos/primary-logo.svg"},{"@type":"WebSite","@id":"https://docs.treasure.ai/#website","name":"Treasure AI Documentation","url":"https://docs.treasure.ai/","inLanguage":["en","ja"],"publisher":{"@id":"https://www.treasure.ai/"}}]},"llmstxt":{"hide":false,"sections":[{"title":"Table of contents","includeFiles":["**/*"],"excludeFiles":[]}],"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"google-bigquery-import-integration-cli","__idx":0},"children":["Google Bigquery Import Integration CLI"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you prefer, you can use the connector via ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://toolbelt.treasuredata.com/"},"children":["TD Toolbelt"]},"."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Set up the TD Toolbelt on the CLI."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"create-configuration-file","__idx":1},"children":["Create Configuration File"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Create configuration YAML file that is referred to as \"config.yml\" here."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"example-configyml","__idx":2},"children":["Example (config.yml)"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"in:\n  type: bigquery\n  project_id: my-project\n  auth_method: json_key\n  json_keyfile:\n    content: |\n      {\n        \"type\": \"service_account\",\n        \"project_id\": \"xxxxxx\",\n        ...\n       }\n  import_type: table\n  dataset: my_dataset\n  table: my_table\n  incremental: true\n  incremental_columns: [id]\n  export_to_gcs: true\n  temp_dataset: temp\n  temp_table: temp_table\n  gcs_bucket: my-bucket\n  gcs_path_prefix: data-connector/result-\nout:\n  type: td\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"authentication-of-gcp","__idx":3},"children":["Authentication of GCP"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"json-key","__idx":4},"children":["JSON Key"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Specify \"auth_method: json_key\" and put a JSON content of your service account key into \"json_keyfile**.**content\""]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"auth_method: json_key\njson_keyfile:\n  content: |\n    {\n      \"type\": \"service_account\",\n      \"project_id\": \"xxxxxx\",\n      ...\n     } \n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"oauth","__idx":5},"children":["OAuth"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you want to use authorized account by your OAuth 2 application, specify \"auth_method: oauth2\", \"client_id\", \"client_secret\" and \"refresh_token\""]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"auth_method: oauth2\nclient_id: 000000000000-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.apps.googleusercontent.com\nclient_secret: yyyyyyyyyyyyyyyyyyyyyyyy\nrefresh_token: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"import-type","__idx":6},"children":["Import Type"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"table-loading","__idx":7},"children":["Table Loading"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["With table loading, specify \"import_type: table\", \"dataset\" and \"table\""]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"import_type-table--dataset-my_dataset--table-my_table","__idx":8},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["import_type: table"]}," ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["dataset: my_dataset"]}," ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["table: my_table"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"query-loading","__idx":9},"children":["Query Loading"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["With query loading, specify \"import_type: query\" and \"query\""]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"import_type: query\nquery: |-\n  SELECT\n    id, first_name, last_name, created_at\n  FROM\n    my_dataset.my_table\n  WHERE first_name = \"Treasure\"\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can optionally specify \"query_option\". \"use_leagacy_sql\" is ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["false"]}," by default and \"use_query_cache\" is ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["true"]}," by default."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"query: SELECT ...\nquery_option:\n  use_legacy_sql: false\n  use_query_cache: true \n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"data-location","__idx":10},"children":["Data Location"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can specify the location by \"",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["location"]},"\" if necessary"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"location: asia-northeast1\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"incremental-loading","__idx":11},"children":["Incremental Loading"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To enable it, specify \"incremental: true\" and \"incremental_columns\""]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"incremental: true\nincremental_columns: [id]\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"import-large-dataset","__idx":12},"children":["Import Large Dataset"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To enable it, specify \"",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["export_to_gcs: true"]},"\", then add \"temp_dataset\", \"temp_table\", \"gcs_bucket\" and \"gcs_path_prefix\""]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"export_to_gcs: true\ntemp_dataset: temp\ntemp_table: temp_table\ngcs_bucket: my-bucket\ngcs_path_prefix: data-connector/result-\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"optional-preview","__idx":13},"children":["(Optional) Preview"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Run ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["td connector:preview"]}," command to validate your configuration file"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"td connector:preview config.yml\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"create-new-connector-session","__idx":14},"children":["Create New Connector Session"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Run ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["td connector:create"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["By the following example, a daily import session with BigQuery connector is created."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:create daily_bigquery_import \\\n\"10 0 * * *\" td_sample_db td_sample_table config.yml\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"data-partition-key","__idx":15},"children":["Data Partition Key"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Connector sessions need at least one timestamp column in result data to be used as data partition key and the first timestamp column is chosen as the key by default. Use \"",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["--time-column"]},"\" option if you want to explicitly specify a column."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:create --time-column created_at \\\ndaily_bigquery_import ...\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If your result data doesn't have any timestamp column, add the \"",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["time"]},"\" column by adding the filter configuration as follows."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"in:\n  type: bigquery\n  ...\nfilters:\n- type: add_time\n  from_value:\n    mode: upload_time\n  to_column:\n    name: time\nout:\n  type: td\n","lang":"yaml"},"children":[]}]},"headings":[{"value":"Google Bigquery Import Integration CLI","id":"google-bigquery-import-integration-cli","depth":1},{"value":"Create Configuration File","id":"create-configuration-file","depth":2},{"value":"Example (config.yml)","id":"example-configyml","depth":3},{"value":"Authentication of GCP","id":"authentication-of-gcp","depth":3},{"value":"JSON Key","id":"json-key","depth":4},{"value":"OAuth","id":"oauth","depth":4},{"value":"Import Type","id":"import-type","depth":3},{"value":"Table Loading","id":"table-loading","depth":4},{"value":"import_type: table  dataset: my_dataset  table: my_table","id":"import_type-table--dataset-my_dataset--table-my_table","depth":4},{"value":"Query Loading","id":"query-loading","depth":4},{"value":"Data Location","id":"data-location","depth":3},{"value":"Incremental Loading","id":"incremental-loading","depth":3},{"value":"Import Large Dataset","id":"import-large-dataset","depth":3},{"value":"(Optional) Preview","id":"optional-preview","depth":2},{"value":"Create New Connector Session","id":"create-new-connector-session","depth":2},{"value":"Data Partition Key","id":"data-partition-key","depth":3}],"frontmatter":{"seo":{"title":"Google Bigquery Import Integration CLI","description":"Learn how to import data from Google BigQuery using TD Toolbelt CLI, with step-by-step configuration for authentication, table selection, and incremental imports."}},"lastModified":"2026-06-17T07:22:53.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/int/google-bigquery-import-integration-cli","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}