{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":["admonition"]},"redocly_category":"Integrations","type":"markdown"},"seo":{"title":"Google BigQuery Import Integration","description":"Treasure Data Product Documentation · Collect and Unify · Segment and Activate · Experiment and Analyze · Decisioning Automate with AI Scale and Trust.","siteUrl":"https://docs.treasuredata.com","lang":"en-US","llmstxt":{"hide":false,"sections":[{"title":"Table of contents","includeFiles":["**/*"],"excludeFiles":[]}],"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"google-bigquery-import-integration","__idx":0},"children":["Google BigQuery Import Integration"]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"warning","name":"Legacy SQL usage restrictions from June 1, 2026"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Effective ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["June 1, 2026"]},", Google will ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://cloud.google.com/bigquery/docs/legacy-sql-feature-availability"},"children":["limit BigQuery Legacy SQL use"]},". Whether your project can continue running Legacy SQL after that date depends on whether your organization or project used it between ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["November 1, 2025 and June 1, 2026"]},":"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Projects that did not use Legacy SQL during this period will be unable to use it after June 1, 2026."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Projects that did use it can keep running existing workloads, but new Legacy SQL workloads might be blocked."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["We recommend using Standard SQL (GoogleSQL) for new queries and migrating any remaining Legacy SQL queries. See Google's ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://cloud.google.com/bigquery/docs/reference/standard-sql/migrating-from-legacy-sql"},"children":["Migrating from Legacy SQL to GoogleSQL"]}," guide for details."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/int/google-bigquery-export-integration"},"children":["Learn more about Google BigQuery Export Integration"]},"."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The integration for Google BigQuery enables the import of data from your BigQuery tables or from query results into Treasure Data."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can use the same BigQuery connection for both import and export, but cannot use connections authenticated by OAuth for export."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"prerequisites","__idx":1},"children":["Prerequisites"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Basic knowledge of Treasure Data"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Basic knowledge of Google Cloud Platform (BigQuery, Cloud Storage and IAM)"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"limitations","__idx":2},"children":["Limitations"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["OAuth is no longer supported for this connector. Only JSON keyfile is supported."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["If your dataset is in a location other than the US or EU multi-region, you need to specify the location. Otherwise, your job in TD will fail with an error: Cannot find job_id xxxxx."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["This connector doesn't support importing ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://cloud.google.com/bigquery/docs/external-data-sources"},"children":["external tables"]},"."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"permissions-and-roles","__idx":3},"children":["Permissions and Roles"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For use of this data connector, authorized accounts (service account) must have the following permissions or IAM roles."]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Category"]}]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Required permissions"]}]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Minimum IAM roles"]}]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["To use table loading"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["- bigquery.tables.get - bigquery.tables.getData"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["- BigQuery Data Viewer"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["To use query loading"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["- bigquery.jobs.create"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["- BigQuery Job User"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["To use \"Import Large Dataset\""]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["- bigquery.tables.export - bigquery.tables.delete - storage.buckets.get - storage.objects.list - storage.objects.create - storage.objects.delete - storage.objects.get"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["- BigQuery Data Editor - Storage Legacy Bucket Writer - Storage Legacy Object Reader"]}]}]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For more details about IAM permissions and roles see the Google Cloud documents: ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://cloud.google.com/bigquery/docs/access-control"},"children":["BigQuery"]}," and ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://cloud.google.com/storage/docs/access-control/iam-roles"},"children":["Cloud Storage"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"use-treasure-console","__idx":4},"children":["Use Treasure Console"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"create-a-new-authentication","__idx":5},"children":["Create a New Authentication"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Go to ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Integrations Hub"]}," > ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Catalog"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Search and select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Google BigQuery"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["A dialog will open, then choose an authentication mode. Only ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["JSON keyfile"]}," is supported.",{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image-20200306-213111.2f7b81cad307daf517ebc01a4f11e29bde5728083359fa0a683dce94107cdfdf.743d8bf4.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Provide the JSON string of your service account key into the \"JSON keyfile\" section."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["See ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://cloud.google.com/iam/docs/creating-managing-service-account-keys"},"children":["the Google Cloud document"]}," to create a new service account key."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you want to load a SQL result, select \"Query statement\", then input an SQL query into the \"SQL statement\"."," ","Before creating a transfer, confirm that your query is valid in the BigQuery Web UI.  ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.cloud.google.com/bigquery/docs/bigquery-web-ui"},"children":["https://docs.cloud.google.com/bigquery/docs/bigquery-web-ui"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"create-a-new-transfer","__idx":6},"children":["Create a New Transfer"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After creating the connection, you are automatically taken to the Authentications tab. Look for the connection you created and select Source."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"fetch-from","__idx":7},"children":["Fetch from"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Configure the data source to import."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Input an ID of your Google Cloud Platform project into \"Project ID\"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"import-types","__idx":8},"children":["Import Types"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select a type of import, either loading a whole table (",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["table loading"]},") or loading a SQL result (",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["query loading"]},")."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Table Loading"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you want to load a whole table, select \"Table\", then provide the \"Dataset name\" and \"Table name\" that you want to export."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you want to load a materialized view, please select \"Query Statement\" instead."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image-20191017-210451.48450469a4e71f457a4f4e09586b3ed17e1121d210307736f1298c2424bb64f4.743d8bf4.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Query Loading"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you want to load a SQL result, select \"Query statement\", then input an SQL query into the \"SQL statement\"."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image-20191017-210506.8e88a9fcfe00fec730507e3846793d39fd798b55f5223e42895003c7c193d07a.743d8bf4.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The default SQL dialect is ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://cloud.google.com/bigquery/docs/reference/standard-sql/"},"children":["Standard SQL (GoogleSQL)"]},". Check ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Use Legacy SQL"]}," if you want to use ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://cloud.google.com/bigquery/docs/reference/legacy-sql"},"children":["Legacy SQL"]},"."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["By default, this connector uses ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://cloud.google.com/bigquery/docs/cached-results"},"children":["cached result"]}," under specific conditions. Uncheck ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Use Cached Results"]}," if you want to disable caching."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"data-location","__idx":9},"children":["Data Location"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You must specify the location if your data is in a location other than the US or EU multi-region."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You must specify the location when your data in the asia-northeast1 region."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["See the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://cloud.google.com/bigquery/docs/dataset-locations"},"children":["Google Cloud document"]}," for more details about the location."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"incremental-loading","__idx":10},"children":["Incremental Loading"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Incremental loading can load only new records after last execution by using increasing, unique column(s), such as an auto-increment ID column or timestamp column for the created date."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To enable it, check ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Incremental Loading"]},", then specify column names to increment into \"Incremental Column Names.\""," ","Only numerical types (INTEGER and FLOAT) and TIMESTAMP type are supported as an incremental column."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image-20191017-210528.65b7be0bce035dff443851f92f3dca4efe7ae4691f8841f141881024578a1ddb.743d8bf4.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"how-this-works","__idx":11},"children":["How This Works"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["This connector records \"last record\" which is the latest record ordered by the incremental columns. In the next execution, it loads records by running a query built by the following rule using the last record:"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["With table loading, all fields are selected with the WHERE clause."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"sql","header":{"controls":{"copy":{}}},"source":"SELECT\n  *\nFROM\n  `${dataset}.${table}`\nWHERE\n  ${incremental_column} > ${value_of_last_record}\n","lang":"sql"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["With query loading, the raw query is wrapped with the WHERE clause."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"sql","header":{"controls":{"copy":{}}},"source":"SELECT\n  *\nFROM\n  (${query}) embulk_incremental_\nWHERE\n  ${incremental_column} > ${value_of_last_record}\n","lang":"sql"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If there are multiple incremental columns (c1, c2 and c3, for example), the WHERE clause is similar to the following statement:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"sql","header":{"controls":{"copy":{}}},"source":"WHERE\n  (c1 > 1)\nOR\n  (c1 = 1 AND c2 > 2)\nOR\n  (c1 = 1 AND c2 = 2 AND c3 > 3)\n","lang":"sql"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"import-large-dataset","__idx":12},"children":["Import Large Dataset"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["When you load a large dataset (more than 500MB as a benchmark), we recommend that you use this \"Import Large Dataset\" option. This option exports the data as GCS (Google Cloud Storage) objects and loads the data in multiple tasks. Hence, loading is faster."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To enable this option, check ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Import Large Dataset"]}," then specify \"Temp dataset\", \"Temp table\", \"GCS bucket\" and \"GCS path prefix\". The \"Temp dataset\" must be created manually in advance."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image-20191017-210539.75a789acbdc72b1c80e71df4f292be9b96765913ef7ffd85e218da75f9ad8cd0.743d8bf4.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["When running a query (query loading or table loading with incremental loading), the query result is exported to a temporary BigQuery table \"temp.temp_table\"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Then the temp table is exported to \"",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["gs://my-bucket/data-connector/result-[12 digists number].jsonl.gz"]},"\" as gzipped ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"http://jsonlines.org/"},"children":["JSON Lines"]}," files. The number of files depends on the size of result data."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["With a table loading without incremental loading, all the data in the source table is directly exported to GCS."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["After completion, the temp table and GCS objects are deleted."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Temp table must be in the same location as the tables you're querying. See the Google Cloud ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://cloud.google.com/bigquery/docs/writing-results#temporary_and_permanent_tables"},"children":["temporary and permanent tables document"]}," for more details."," ","GCS bucket must be also in the same location as the tables unless the dataset is set to \"US.\" You can export data from a US-based dataset to a Cloud Storage bucket in another region. See the Google Cloud ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://cloud.google.com/bigquery/docs/exporting-data#export_limitations"},"children":["export limitations document"]}," for more details."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"data-preview","__idx":13},"children":["Data Preview"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can see a ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/integration-hub/batch/import/previewing-your-source-data"},"children":["preview"]}," of your data before running the import by selecting Generate Preview. Data preview is optional and you can safely skip to the next page of the dialog if you choose to."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next"]},". The Data Preview page opens."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["If you want to preview your data, select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Generate Preview"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Verify the data."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"data-placement","__idx":14},"children":["Data Placement"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For data placement, select the target database and table where you want your data placed and indicate how often the import should run."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next."]}," Under Storage, you will create a new or select an existing database and create a new or select an existing table for where you want to place the imported data."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select a ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Database"]}," > ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Select an existing"]}," or ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create New Database"]},"."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Optionally, type a database name."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select a ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Table"]},"> ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Select an existing"]}," or ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create New Table"]},"."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Optionally, type a table name."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Choose the method for importing the data."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Append"]}," (default)-Data import results are appended to the table."," ","If the table does not exist, it will be created."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Always Replace"]},"-Replaces the entire content of an existing table with the result output of the query. If the table does not exist, a new table is created."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Replace on New Data"]},"-Only replace the entire content of an existing table with the result output when there is new data."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Timestamp-based Partition Key"]}," column."," ","If you want to set a different partition key seed than the default key, you can specify the long or timestamp column as the partitioning time. As a default time column, it uses upload_time with the add_time filter."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Timezone"]}," for your data storage."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Under ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Schedule"]},", you can choose when and how often you want to run this query."]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"run-once","__idx":15},"children":["Run once"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Off"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Scheduling Timezone"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create & Run Now"]},"."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"repeat-regularly","__idx":16},"children":["Repeat Regularly"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["On"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Schedule"]},". The UI provides these four options: ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["@hourly"]},", ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["@daily"]}," and ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["@monthly"]}," or custom ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["cron"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You can also select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Delay Transfer"]}," and add a delay of execution time."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Scheduling Timezone"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create & Run Now"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After your transfer has run, you can see the results of your transfer in ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Data Workbench"]}," > ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Databases."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"data-conversion","__idx":17},"children":["Data Conversion"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["BigQuery's data types are automatically converted to a corresponding Treasure Data type, as indicated in the following table. If you include unsupported types in the schema of the table or query result, you receive errors."]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["BigQuery"]}]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Treasure Data"]}]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["STRING"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["string"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["BYTES"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"em","attributes":{},"children":["Unsupported"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["INTEGER"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["long"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["FLOAT"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["double"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["NUMERIC"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"em","attributes":{},"children":["Unsupported"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["BOOLEAN"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["long (true is 1, false is 0)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["TIMESTAMP"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["string (yyyy-MM-dd HH:mm:ss.SSS)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["DATE"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"em","attributes":{},"children":["Unsupported"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["TIME"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"em","attributes":{},"children":["Unsupported"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["DATETIME"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"em","attributes":{},"children":["Unsupported"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["RECORD"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["string (as JSON)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["REPEATED (PRIMITIVE or RECORD)"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["string (as JSON)"]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"use-connections-for-input-and-output","__idx":18},"children":["Use Connections for Input and Output"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can use the same BigQuery connection for both Data Connector (input) and Result Output (output), but, currently, cannot use connections authenticated by OAuth for output."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"quotas-and-limits","__idx":19},"children":["Quotas and Limits"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Any quotas and limits of ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://cloud.google.com/bigquery/quotas"},"children":["BigQuery and Cloud Storage"]}," are applied to your GCP project."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"use-bigquery-connector-via-cli","__idx":20},"children":["Use BigQuery Connector via CLI"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you prefer, you can use the connector via ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://toolbelt.treasuredata.com/"},"children":["TD Toolbelt"]},"."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Set up ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/tools/cli-and-sdks/quickstart"},"children":["TD"]}," ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/tools/cli-and-sdks/quickstart"},"children":["Tool"]},{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/tools/cli-and-sdks/quickstart"},"children":["belt on the CLI"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"create-configuration-file","__idx":21},"children":["Create Configuration File"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Create configuration YAML file that is referred to as \"config.yml\" here."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"example-configyml","__idx":22},"children":["Example (config.yml)"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"in:\n  type: bigquery\n  project_id: my-project\n  auth_method: json_key\n  json_keyfile:\n    content: |\n      {\n        \"type\": \"service_account\",\n        \"project_id\": \"xxxxxx\",\n        ...\n       }\n  import_type: table\n  dataset: my_dataset\n  table: my_table\n  incremental: true\n  incremental_columns: [id]\n  export_to_gcs: true\n  temp_dataset: temp\n  temp_table: temp_table\n  gcs_bucket: my-bucket\n  gcs_path_prefix: data-connector/result-\nout:\n  type: td\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"authentication-of-gcp","__idx":23},"children":["Authentication of GCP"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"json-key","__idx":24},"children":["JSON Key"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Specify \"auth_method: json_key\" and put a JSON content of your service account key into \"json_keyfile**.**content\""]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"auth_method: json_key\njson_keyfile:\n  content: |\n    {\n      \"type\": \"service_account\",\n      \"project_id\": \"xxxxxx\",\n      ...\n     } \n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"oauth","__idx":25},"children":["OAuth"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you want to use authorized account by your OAuth 2 application, specify \"auth_method: oauth2\", \"client_id\", \"client_secret\" and \"refresh_token\""]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"auth_method: oauth2\nclient_id: 000000000000-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.apps.googleusercontent.com\nclient_secret: yyyyyyyyyyyyyyyyyyyyyyyy\nrefresh_token: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"import-type","__idx":26},"children":["Import Type"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"table-loading","__idx":27},"children":["Table Loading"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["With table loading, specify \"import_type: table\", \"dataset\" and \"table\""]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"import_type-table--dataset-my_dataset--table-my_table","__idx":28},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["import_type: table"]}," ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["dataset: my_dataset"]}," ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["table: my_table"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"query-loading","__idx":29},"children":["Query Loading"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["With query loading, specify \"import_type: query\" and \"query\""]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"import_type: query\nquery: |-\n  SELECT\n    id, first_name, last_name, created_at\n  FROM\n    my_dataset.my_table\n  WHERE first_name = \"Treasure\"\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can optionally specify \"query_option\". \"use_leagacy_sql\" is ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["false"]}," by default and \"use_query_cache\" is ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["true"]}," by default."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"query: SELECT ...\nquery_option:\n  use_legacy_sql: false\n  use_query_cache: true \n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"data-location-1","__idx":30},"children":["Data Location"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can specify the location by \"",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["location"]},"\" if necessary"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"location: asia-northeast1\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"incremental-loading-1","__idx":31},"children":["Incremental Loading"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To enable it, specify \"incremental: true\" and \"incremental_columns\""]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"incremental: true\nincremental_columns: [id]\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"import-large-dataset-1","__idx":32},"children":["Import Large Dataset"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To enable it, specify \"",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["export_to_gcs: true"]},"\", then add \"temp_dataset\", \"temp_table\", \"gcs_bucket\" and \"gcs_path_prefix\""]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"export_to_gcs: true\ntemp_dataset: temp\ntemp_table: temp_table\ngcs_bucket: my-bucket\ngcs_path_prefix: data-connector/result-\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"optional-preview","__idx":33},"children":["(Optional) Preview"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Run td ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/tools/cli-and-sdks/api#td-connector-preview"},"children":["td connector:preview"]}," command to validate your configuration file"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td connector:preview config.yml\n+---------+-------------------+------------------+-------------------------------+\n| id:long | first_name:string | last_name:string | created_at:timestamp          |\n+---------+-------------------+------------------+-------------------------------+\n| 1       | \"Treasure\"        | \"Data\"           | \"2018-05-21 12:00:00.111 UTC\" |\n+---------+-------------------+------------------+-------------------------------+\n1 row in set\nUpdate config.yml and use 'td connector:preview config.yml' to preview again.\nUse 'td connector:issue config.yml' to run Server-side bulk load.\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"create-new-connector-session","__idx":34},"children":["Create New Connector Session"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Run ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/tools/cli-and-sdks/api#td-connector-create"},"children":["td connector:create"]},"."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["By the following example, a daily import session with BigQuery connector is created."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td connector:create daily_bigquery_import \\\n    \"10 0 * * *\" td_sample_db td_sample_table config.yml\nName     : daily_bigquery_import\nCron     : 10 0 * * *\nTimezone : UTC\nDelay    : 0\nDatabase : td_sample_db\nTable    : td_sample_table\nConfig\n---\nin:\n  ...\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"data-partition-key","__idx":35},"children":["Data Partition Key"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Connector sessions need at least one timestamp column in result data to be used as data partition key and the first timestamp column is chosen as the key by default. Use \"",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["--time-column"]},"\" option if you want to explicitly specify a column."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td connector:create --time-column created_at \\\n    daily_bigquery_import ...\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If your result data doesn't have any timestamp column, add the \"",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["time"]},"\" column by adding the filter configuration as follows."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"in:\n  type: bigquery\n  ...\nfilters:\n- type: add_time\n  from_value:\n    mode: upload_time\n  to_column:\n    name: time\nout:\n  type: td\n"},"children":[]}]},"headings":[{"value":"Google BigQuery Import Integration","id":"google-bigquery-import-integration","depth":1},{"value":"Prerequisites","id":"prerequisites","depth":2},{"value":"Limitations","id":"limitations","depth":2},{"value":"Permissions and Roles","id":"permissions-and-roles","depth":2},{"value":"Use Treasure Console","id":"use-treasure-console","depth":2},{"value":"Create a New Authentication","id":"create-a-new-authentication","depth":3},{"value":"Create a New Transfer","id":"create-a-new-transfer","depth":3},{"value":"Fetch from","id":"fetch-from","depth":4},{"value":"Import Types","id":"import-types","depth":3},{"value":"Data Location","id":"data-location","depth":3},{"value":"Incremental Loading","id":"incremental-loading","depth":3},{"value":"How This Works","id":"how-this-works","depth":4},{"value":"Import Large Dataset","id":"import-large-dataset","depth":3},{"value":"Data Preview","id":"data-preview","depth":3},{"value":"Data Placement","id":"data-placement","depth":3},{"value":"Run once","id":"run-once","depth":4},{"value":"Repeat Regularly","id":"repeat-regularly","depth":4},{"value":"Data Conversion","id":"data-conversion","depth":2},{"value":"Use Connections for Input and Output","id":"use-connections-for-input-and-output","depth":2},{"value":"Quotas and Limits","id":"quotas-and-limits","depth":2},{"value":"Use BigQuery Connector via CLI","id":"use-bigquery-connector-via-cli","depth":2},{"value":"Create Configuration File","id":"create-configuration-file","depth":3},{"value":"Example (config.yml)","id":"example-configyml","depth":4},{"value":"Authentication of GCP","id":"authentication-of-gcp","depth":3},{"value":"JSON Key","id":"json-key","depth":4},{"value":"OAuth","id":"oauth","depth":4},{"value":"Import Type","id":"import-type","depth":3},{"value":"Table Loading","id":"table-loading","depth":4},{"value":"import_type: table  dataset: my_dataset  table: my_table","id":"import_type-table--dataset-my_dataset--table-my_table","depth":4},{"value":"Query Loading","id":"query-loading","depth":4},{"value":"Data Location","id":"data-location-1","depth":3},{"value":"Incremental Loading","id":"incremental-loading-1","depth":3},{"value":"Import Large Dataset","id":"import-large-dataset-1","depth":3},{"value":"(Optional) Preview","id":"optional-preview","depth":2},{"value":"Create New Connector Session","id":"create-new-connector-session","depth":2},{"value":"Data Partition Key","id":"data-partition-key","depth":3}],"frontmatter":{"seo":{"title":"Google BigQuery Import Integration"}},"lastModified":"2026-06-01T09:09:59.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/int/google-bigquery-import-integration","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}