{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":[]},"redocly_category":"Integrations","type":"markdown"},"seo":{"title":"Google Cloud Storage Import V2 Integration","description":"Treasure Data Product Documentation · Collect and Unify · Segment and Activate · Experiment and Analyze · Decisioning Automate with AI Scale and Trust.","siteUrl":"https://docs.treasuredata.com","lang":"en-US","llmstxt":{"hide":false,"sections":[{"title":"Table of contents","includeFiles":["**/*"],"excludeFiles":[]}],"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"google-cloud-storage-import-v2-integration","__idx":0},"children":["Google Cloud Storage Import V2 Integration"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The Data Connector for Google Cloud Storage enables import of the contents of ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["*.tsv"]}," and ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["*.csv"]}," files stored in your GCS bucket. For export Integration, see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/int/google-cloud-storage-export-v2-integration"},"children":["Google Cloud Storage Export V2 Integration"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"prerequisites","__idx":1},"children":["Prerequisites"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Basic knowledge of Treasure Data, including ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://toolbelt.treasuredata.com/"},"children":["TD Toolbelt"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["A Google Cloud Platform account with specific permissions"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"static-ip-address-of-treasure-data-integration","__idx":2},"children":["Static IP Address of Treasure Data Integration"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If your security policy requires IP whitelisting, you must add Treasure Data's IP addresses to your allowlist to ensure a successful connection."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Please find the complete list of static IP addresses, organized by region, at the following ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/apis/endpoints/ip-addresses-integrations-result-workers"},"children":["document"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"obtain-the-google-json-credentials","__idx":3},"children":["Obtain the Google JSON Credentials"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The integration with Google Cloud Storage is based on server-to-server API authentication."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The Service Account used to generate the JSON Credentials must have Storage Object User permissions."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Visit your Google Developer Console."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select Credentials under APIs & Services at the left menu."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gcs_v2_2.d96a9f7dec347056740ae90fc6866fbc6c78f9d9b5d1a999dc0a3663452e4f1f.01768ec3.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":3},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select Create credentials then select Service account:"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gcs_v2_3.6a564dc9b561061e0b5fa0f783c196f3585cd3309c397bd56f2e713dce828210.01768ec3.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":4},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["From Permissions add Storage Object User Role."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"obtain-application-default-credentials-adc-keyfile","__idx":4},"children":["Obtain Application Default Credentials (ADC) Keyfile"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select create pool under IAM & Admin / Workload Identity Pools or select existing Pool"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gcs_v2_5.b29c6dbd8144a5c1208953a7129b820e778c43284075c479a9ebd04dc834d2a8.01768ec3.png","alt":""},"children":[]}," ",{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gcs_v2_4.8c6962bc70473778e721ddf4e7e6ae5c7c2e9cb26bc8510be8f55a328670d354.01768ec3.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":2},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Add an AWS provider with account ID ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["523683666290"]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gcs_v2_6.3e3283ca2462b5e616ca1ffa6d5b2cc5a76b0b6327e096428c01db5d5cb40652.01768ec3.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":3},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["From Configure provider attributes select Add mapping. Add attribute with name ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["attribute.account"]}," and value ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["assertion.account"]}," then Save"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["From Workload Identity Pools select created pool then select Download config from Connected service accounts"," ",{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gcs_v2_7.9399e7b93f7f9b6239278e8f87af939fc11f0fd1c06199569a5394340e6090c3.01768ec3.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select AWS provider that created for account ID ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["523683666290"]}," then select Download and store config file (Application default credential keyfile)."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"grant-aws-provider-permission-to-access-destination-bucket","__idx":5},"children":["Grant AWS provider permission to access destination bucket"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select destination bucket from your buckets list then select permissions tab"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select on Grant access then add Principals value ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["principalSet://iam.googleapis.com/projects/{PROJECT_NUMBER}/locations/global/workloadIdentityPools/{POOL_ID}/attribute.account/523683666290"]}," with roles Storage Object User"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gcs_v2_10.2c69560a43e6aa632bf83ee8b04441e42c3fbd8926f1c820cfc9c0f44116736a.01768ec3.png","alt":""},"children":[]}," ",{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gcs_v2_11.61fd4999a78444d32248ae89c67881555f3005de4825f1cb24b01b129f8b35c1.01768ec3.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"create-a-new-authentication","__idx":6},"children":["Create a New Authentication"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["When you configure a data connection, you provide authentication to access the integration. In Treasure Data, you configure the authentication and then specify the source information."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Open ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Treasure Console"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Navigate to ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Integrations Hub"]}," > ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Catalog"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Search for and select Google Cloud Storage V2."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gcs_v2_12.4c53928180751cb2fe12c8abe6bbd276c87d3670b1bb12c494840f482ad9caa7.01768ec3.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":4},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create Authentication"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select Authentication Method and input the credentials."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gcs_v2_13.cb9ab21dd436bcf45f8846088a5ffa3e8ef4b4c42592817d868907667d46bf4c.01768ec3.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":6},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Type a name for your connection."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Continue"]},"."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"transfer-your-google-cloud-storage-data-to-treasure-data","__idx":7},"children":["Transfer Your Google Cloud Storage Data to Treasure Data"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After creating the authenticated connection, you are automatically taken to Authentications."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Search for the connection you created."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["New Source"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Type a name for your ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Source"]}," in the Data Transfer field."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gcs_v2_14.12ab6edfe6657b89779d7b2982d231fce69441119e7b234f7e29c32899386dbf.01768ec3.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"source-table","__idx":8},"children":["Source Table"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["The Source Table dialog opens. Edit the following parameters"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gcs_v2_15.cc6c89ab4619ad878d7d9426681bb8f9aa6f6dce0154f6cdb7c4f4caa99bf35b.01768ec3.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Parameters"]}]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Description"]}]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Bucket"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Google Cloud Storage bucket name (Ex. ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["your_bucket_name"]},")"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Path Prefix"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Prefix of target keys. (Ex. ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["logs/data_"]},")"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Path Regex"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["regexp to match file paths. If a file path doesn’t match with this pattern, the file is skipped. (Ex. ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":[".csv$"]}," # in this case, a file is skipped if its path doesn’t match with this pattern.)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Start after path"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Inserts ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["last_path"]}," parameter so that the first execution skips files before the path. (Ex. ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["logs/data_20170101.csv"]},")"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Incremental"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Enables incremental loading. If incremental loading is enabled, config diff for the next execution will include ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["last_path"]}," parameter so that next execution skips files before the path. Otherwise, ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["last_path"]}," will not be included."]}]}]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["In this case, the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Source Table"]}," setting should be as shown:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Bucket"]},": your_bucket"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Path Prefix"]},": path/to/name"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Path Regex"]},": ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":[".csv$"]}," (Not Required)"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Start after path"]},": path/to/name1.csv (Assuming that you want to import the files after name1.csv)"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Incremental"]},": true (If you want to schedule this job.)"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"data-settings","__idx":9},"children":["Data Settings"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["The ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Data Settings"]}," page opens."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Optionally, edit the data settings or skip this page of the dialog."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gcs_v2_16.d8cb1d6af994f3003283498c0e8f4fdd0616979d4c642b7f1d77133672b71b99.01768ec3.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/gcs_v2_17.50f24e1fb8d74b8ce877136a26f616367cb488af0944ab631593ae069465fa36.01768ec3.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"parser","__idx":10},"children":["Parser"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Parameters"]}]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Description"]}]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Type"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Following types are supported:   - Avro - CSV - json - Query String"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Default timezone"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Changes time zone of timestamp columns if the value itself doesn’t include time zone."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Total file count limit"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Maximum number of files to read. (optional)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Schema Settings"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["You can name the columns and set the data type. TD parses a value as a specified data type. And then, it stores after converting to Treasure Data schema. Support values:",{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["boolean"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["long"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["timestamp"]},": will be imported as String type at Treasure Data (Ex. 2017-04-01 00:00:00.000)"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["double"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["string"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["json"]}]}]}]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"data-preview","__idx":11},"children":["Data Preview"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can see a ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/integration-hub/batch/import/previewing-your-source-data"},"children":["preview"]}," of your data before running the import by selecting Generate Preview. Data preview is optional and you can safely skip to the next page of the dialog if you choose to."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next"]},". The Data Preview page opens."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["If you want to preview your data, select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Generate Preview"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Verify the data."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"data-placement","__idx":12},"children":["Data Placement"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For data placement, select the target database and table where you want your data placed and indicate how often the import should run."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next."]}," Under Storage, you will create a new or select an existing database and create a new or select an existing table for where you want to place the imported data."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select a ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Database"]}," > ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Select an existing"]}," or ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create New Database"]},"."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Optionally, type a database name."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select a ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Table"]},"> ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Select an existing"]}," or ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create New Table"]},"."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Optionally, type a table name."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Choose the method for importing the data."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Append"]}," (default)-Data import results are appended to the table."," ","If the table does not exist, it will be created."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Always Replace"]},"-Replaces the entire content of an existing table with the result output of the query. If the table does not exist, a new table is created."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Replace on New Data"]},"-Only replace the entire content of an existing table with the result output when there is new data."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Timestamp-based Partition Key"]}," column."," ","If you want to set a different partition key seed than the default key, you can specify the long or timestamp column as the partitioning time. As a default time column, it uses upload_time with the add_time filter."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Timezone"]}," for your data storage."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Under ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Schedule"]},", you can choose when and how often you want to run this query."]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"run-once","__idx":13},"children":["Run once"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Off"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Scheduling Timezone"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create & Run Now"]},"."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"repeat-regularly","__idx":14},"children":["Repeat Regularly"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["On"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Schedule"]},". The UI provides these four options: ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["@hourly"]},", ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["@daily"]}," and ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["@monthly"]}," or custom ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["cron"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You can also select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Delay Transfer"]}," and add a delay of execution time."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Scheduling Timezone"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create & Run Now"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After your transfer has run, you can see the results of your transfer in ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Data Workbench"]}," > ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Databases."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"import-from-google-cloud-storage-using-workflow","__idx":15},"children":["Import from Google Cloud Storage using Workflow"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For sample workflows importing data from GCS, view the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://github.com/treasure-data/treasure-boxes/tree/master/td_load/gcs"},"children":["Treasure Boxes"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"import-from-google-cloud-storage-via-cli-toolbelt","__idx":16},"children":["Import from Google Cloud Storage via CLI (Toolbelt)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Before setting up the connector, install the most current TD Toolbelt."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The Data Connector supports parsing of \"boolean\", \"long\", \"double\", \"string\", and \"timestamp\" types."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"create-seed-config-file-seedyml","__idx":17},"children":["Create Seed Config File (seed.yml)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Prepare ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["seed.yml"]}," , with your JSON keyfile. You must also specify bucket name and target file name (or prefix for multiple files)."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["With Service Account authentication"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"in:\n  type: gcs_v2\n  bucket: sample_bucket\n  path_prefix: path/to/sample_file  # path to the *.csv or *.tsv file on your GCS bucket\n  path_match_pattern: \"*.csv.gz$\"\n  auth_method: json_key\n  json_keyfile:\n    content: |\n      {\n        \"private_key_id\": \"1234567890\",\n        \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nABCDEF\",\n        \"client_id\": \"...\",\n        \"client_email\": \"...\",\n        \"type\": \"service_account\"\n      }\nout:\n  mode: append\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["With Workload Identity Federation authentication"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"in:\n  type: gcs_v2\n  bucket: sample_bucket\n  path_prefix: path/to/sample_file  # path to the *.csv or *.tsv file on your GCS bucket\n  path_match_pattern: \"*.csv.gz$\"\n  auth_method: wif\n  adc_keyfile:\n    content: |\n      {\n        ..........\n      }\nout:\n  mode: append\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The Data Connector for Google Cloud Storage imports all files that match a specified prefix. (e.g. path_prefix: ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_"]}," –> ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201501.csv.gz"]},", ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201502.csv.gz"]},", …, ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201505.csv.gz"]},")"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"guess-fields-generate-loadyml","__idx":18},"children":["Guess Fields (Generate load.yml)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Use ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["connector:guess"]},". This command automatically reads the target file, and intelligently guesses the file format."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:guess seed.yml -o load.yml\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you open up ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["load.yml"]},", you see the guessed file format definitions including file formats, encodings, column names, and types."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"in:\n  type: gcs_v2\n  bucket: sample_bucket\n  path_prefix: path/to/sample_file\n  auth_method: json_key\n  json_keyfile:\n    content: |\n      {\n        \"private_key_id\": \"1234567890\",\n        \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nABCDEF\",\n        \"client_id\": \"...\",\n        \"client_email\": \"...\",\n        \"type\": \"service_account\"\n      }\n  decoders:\n  - {type: gzip}\n  parser:\n    charset: UTF-8\n    newline: CRLF\n    type: csv\n    delimiter: ','\n    quote: '\"'\n    escape: ''\n    skip_header_lines: 1\n    columns:\n    - name: id\n      type: long\n    - name: company\n      type: string\n    - name: customer\n      type: string\n    - name: created_at\n      type: timestamp\n      format: '%Y-%m-%d %H:%M:%S'\nout:\n  mode: append\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can preview how the system parses the file by using the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["preview"]}," command."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:preview load.yml\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The guess command needs over 3 rows and 2 columns in source data file, because it guesses column definition using sample rows from source data."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If the system detects your column name or column type unexpectedly, modify ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["load.yml"]}," directly and preview again."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The preview command will download one file from the specified bucket and display the results from that file. This may cause a difference in results from the preview and issue commands."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"execute-load-job","__idx":19},"children":["Execute Load Job"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Submit the load job. It may take a couple of hours depending on the size of the data. Users need to specify the database and table where their data is stored."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["It’s also recommended to specify ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["--time-column"]}," option since Treasure Data’s storage is partitioned by time. If the option is not given, the Data Connector will choose the first ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["long"]}," or ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["timestamp"]}," column as the partitioning time. The type of the column specified by ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["--time-column"]}," must be either of ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["long"]}," or ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["timestamp"]}," type."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If your data doesn’t have a time column, you may add it using the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["add_time"]}," filter option. More details at ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.treasuredata.com/smart/project-product-documentation/add_time-filter-function"},"children":["add_time filter plugin"]},"."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:issue load.yml --database td_sample_db --table td_sample_table   --time-column created_at\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The preceding command assumes you have already created a ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["database(td_sample_db)"]}," and a ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["table(td_sample_table)"]},". If the database or the table do not exist in TD, this command will not succeed so create the database and table manually or use ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["--auto-create-table"]}," option with ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td connector:issue"]}," command to auto-create the database and table:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:issue load.yml --database td_sample_db --table td_sample_table --time-column created_at --auto-create-table\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The Data Connector does not sort records server-side. To use time-based partitioning effectively, sort records in files beforehand."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you have a field called ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["time"]},", you don’t have to specify the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["--time-column"]}," option."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:issue load.yml --database td_sample_db --table td_sample_table\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"scheduled-execution","__idx":20},"children":["Scheduled Execution"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can schedule a periodic Data Connector execution for incremental Google Cloud Storage file imports. TD manages the scheduler carefully to ensure high availability. By using this feature, you no longer need a ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["cron"]}," daemon on your local data center."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For the scheduled import, the Data Connector for Google Cloud Storage imports all files that match with the specified prefix (e.g. path_prefix: ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_"]}," –> ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201501.csv.gz"]},", ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201502.csv.gz"]},", …, ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201505.csv.gz"]},") at first and remembers the last path (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201505.csv.gz"]},") for the next execution."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["On the second and subsequent runs, it will only import files that come after the last path in alphabetical (lexicographic) order. (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201506.csv.gz"]},", …)"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"create-the-schedule","__idx":21},"children":["Create the Schedule"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["A new schedule can be created using the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td connector:create"]}," command. The following are required: the name of the schedule, the cron-style schedule, the database and table where their data will be stored, and the Data Connector configuration file."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:create daily_import \"10 0 * * *\" td_sample_db td_sample_table load.yml\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["It’s also recommended to specify the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["--time-column"]}," option, since Treasure Data’s storage is partitioned by time."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:create daily_import \"10 0 * * *\" td_sample_db td_sample_table load.yml --time-column created_at\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["cron"]}," parameter also accepts three special options: ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["@hourly"]},", ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["@daily"]}," and ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["@monthly"]},". By default, schedule is setup in UTC timezone. You can set the schedule in a timezone using -t or --timezone option. The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["--timezone"]}," option only supports extended timezone formats like 'Asia/Tokyo', 'America/Los_Angeles' etc. Timezone abbreviations like PST, CST are *not* supported and may lead to unexpected schedules."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"list-the-schedules","__idx":22},"children":["List the Schedules"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can see the list of currently scheduled entries by running the command ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td connector:list"]},"."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td connector:list\n"},"children":[]}]},"headings":[{"value":"Google Cloud Storage Import V2 Integration","id":"google-cloud-storage-import-v2-integration","depth":1},{"value":"Prerequisites","id":"prerequisites","depth":2},{"value":"Static IP Address of Treasure Data Integration","id":"static-ip-address-of-treasure-data-integration","depth":2},{"value":"Obtain the Google JSON Credentials","id":"obtain-the-google-json-credentials","depth":3},{"value":"Obtain Application Default Credentials (ADC) Keyfile","id":"obtain-application-default-credentials-adc-keyfile","depth":3},{"value":"Grant AWS provider permission to access destination bucket","id":"grant-aws-provider-permission-to-access-destination-bucket","depth":3},{"value":"Create a New Authentication","id":"create-a-new-authentication","depth":3},{"value":"Transfer Your Google Cloud Storage Data to Treasure Data","id":"transfer-your-google-cloud-storage-data-to-treasure-data","depth":3},{"value":"Source Table","id":"source-table","depth":3},{"value":"Data Settings","id":"data-settings","depth":3},{"value":"Parser","id":"parser","depth":3},{"value":"Data Preview","id":"data-preview","depth":3},{"value":"Data Placement","id":"data-placement","depth":3},{"value":"Run once","id":"run-once","depth":4},{"value":"Repeat Regularly","id":"repeat-regularly","depth":4},{"value":"Import from Google Cloud Storage using Workflow","id":"import-from-google-cloud-storage-using-workflow","depth":2},{"value":"Import from Google Cloud Storage via CLI (Toolbelt)","id":"import-from-google-cloud-storage-via-cli-toolbelt","depth":2},{"value":"Create Seed Config File (seed.yml)","id":"create-seed-config-file-seedyml","depth":3},{"value":"Guess Fields (Generate load.yml)","id":"guess-fields-generate-loadyml","depth":3},{"value":"Execute Load Job","id":"execute-load-job","depth":3},{"value":"Scheduled Execution","id":"scheduled-execution","depth":3},{"value":"Create the Schedule","id":"create-the-schedule","depth":3},{"value":"List the Schedules","id":"list-the-schedules","depth":3}],"frontmatter":{"seo":{"title":"Google Cloud Storage Import V2 Integration"}},"lastModified":"2026-06-01T09:09:59.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/int/google-cloud-storage-import-v2-integration","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}