{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":["admonition"]},"redocly_category":"Integrations","type":"markdown"},"seo":{"title":"SFTP V2 Server Import Integration","description":"Treasure Data Product Documentation · Collect and Unify · Segment and Activate · Experiment and Analyze · Decisioning Automate with AI Scale and Trust.","siteUrl":"https://docs.treasuredata.com","lang":"en-US","llmstxt":{"hide":false,"sections":[{"title":"Table of contents","includeFiles":["**/*"],"excludeFiles":[]}],"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"sftp-v2-server-import-integration","__idx":0},"children":["SFTP V2 Server Import Integration"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The Data Connector for ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["SFTP_V2"]}," enables you to import files stored on your SFTP server to Treasure Data."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"prerequisites","__idx":1},"children":["Prerequisites"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Basic knowledge of Treasure Data."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Before using this integration, determine valid protocols for your environment."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you intend to use ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["SFTP"]},", you can use this integration for SFTP."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If using ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["FTP"]},"/",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["FTPS"]},", try connecting with the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/int/ftp-server-import-integration"},"children":["FTP Import Integration"]},"."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Check your accepted IP range and port if you are using a firewall. Server administrators sometimes change the default port number from TCP 22 for security reasons."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["\"PuTTY\" and other formats are not supported."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"limitations-and-supported","__idx":2},"children":["Limitations and Supported"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Support only the STORED and DEFLATE compression methods."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Multi-part gzip file may not work."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"static-ip-address-of-treasure-data-integration","__idx":3},"children":["Static IP Address of Treasure Data Integration"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If your security policy requires IP whitelisting, you must add Treasure Data's IP addresses to your allowlist to ensure a successful connection."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Please find the complete list of static IP addresses, organized by region, at the following ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/apis/endpoints/ip-addresses-integrations-result-workers"},"children":["document"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"use-the-treasure-console-to-create-your-connection","__idx":4},"children":["Use the Treasure Console to Create Your Connection"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"create-a-new-connection","__idx":5},"children":["Create a New Connection"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["In Treasure Data, you must create and configure the data connection prior to running your query. As part of the data connection, you provide authentication to access the integration."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Open ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Treasure Console"]},"."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Navigate to ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Integrations Hub"]}," >  ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Catalog"]},"."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Search for and select SFTP_V2."]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image2021-9-17_23-49-47.29a282eeb76e1632f676d6e4277e50d179e81310b1a0fd3eef456bd764f48466.8d4637e9.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":4},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create Authentication"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The following dialog opens. Edit the parameters. Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Continue"]},"."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image2021-9-17_23-53-17.c0b0922f8f427a0a6bbf7817123a43acc9ca8e12aba66472879f34e594e1c64f.8d4637e9.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Parameters"},"children":["Parameters"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Description"},"children":["Description"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Host"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The host information of the remote SFTP instance, for example, an IP address."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Port"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The connection port on the remote SFTP instance, the default is 22."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["User"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The user name used to connect to the remote SFTP instance."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Authentication mode"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The way you choose to authenticate with your SFTP server."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Secret key file"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Required if 'public/private key pair' is selected from ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["Authentication Mode"]},". (RSA, DSS, ECDSA, and ED25519 are supported.)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Passphrase for secret key file"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["(Optional) If required, provide a passphrase for the provided secret file."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Retry limit"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The number of times to retry a failed connection (default 10)."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Timeout"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Connection timeout in seconds (default 600)."]}]}]}]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":6},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Enter a name for your connection."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Choose to share the authentication with others or not."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Continue."]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"transfer-your-data-to-treasure-data","__idx":6},"children":["Transfer Your Data to Treasure Data"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After creating the authenticated connection, you are automatically taken to Authentications."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Search for the connection you created."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["New Source"]},"."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Type a name for your ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Source"]}," in the Data Transfer Name field."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image2021-9-18_0-9-29.0b72284e0831e9e9b4ccbf4093d2e06b7f7b3a5597c340ad1420d6998b9ae741.8d4637e9.png","alt":""},"children":[]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next"]},"."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image2021-9-18_0-12-21.cc2a19e06088fb2adcb88ab7f9943ee81469e4fa3eed312dd611bea7504b5ee2.8d4637e9.png","alt":""},"children":[]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Edit the following parameters:"]}]}]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Parameters"},"children":["Parameters"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Description"},"children":["Description"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["User directory root"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Check if the path prefix is under the user directory Ex: /home/test_user"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Path prefix"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Prefix of target files, and it must point to a folder (string, required). Unlike with SFTP v1, the path prefix has to be a folder path. If a partial file name is included in the file path, you will receive an ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["invalid path_prefix:xxx"]}," error message."]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"warning"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you are migrating from SFTP v1 to SFTP v2 Import, note that path_prefix in v2 does not behave the same way as v1. For example, unlike SFTP v.1, the path prefix must be a folder path."]}]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Path match pattern"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Type a regular expression to query file paths. If a file path doesn't match the specified pattern, the file is skipped. For example, if you specify the pattern ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":[".csv$"]},", then a file is skipped if its path doesn't match the pattern."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Incremental"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Enables incremental loading (boolean, optional. default: true). If incremental loading is enabled, the config diff for the next execution will include last_path parameter so that the next execution skips files before the path. Otherwise, last_path is not included."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Start after path"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Only paths lexicographically greater than this will be imported."]}]}]}]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":6},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The Data Settings page can be modified for your needs or you can skip the page."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image2021-9-18_0-17-24.f0bf571ba19bc2d69174a85a689b6c484e762759abf446ea696f96ba5cc04b5d.8d4637e9.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image2023-08-17.8bf4488f6a24c023564fb641a83d36fe324d065e65988518165f784c5ebd1520.8d4637e9.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"data-preview","__idx":7},"children":["Data Preview"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can see a ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/integration-hub/batch/import/previewing-your-source-data"},"children":["preview"]}," of your data before running the import by selecting Generate Preview. Data preview is optional and you can safely skip to the next page of the dialog if you choose to."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next"]},". The Data Preview page opens."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["If you want to preview your data, select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Generate Preview"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Verify the data."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"data-placement","__idx":8},"children":["Data Placement"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For data placement, select the target database and table where you want your data placed and indicate how often the import should run."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next."]}," Under Storage, you will create a new or select an existing database and create a new or select an existing table for where you want to place the imported data."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select a ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Database"]}," > ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Select an existing"]}," or ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create New Database"]},"."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Optionally, type a database name."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select a ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Table"]},"> ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Select an existing"]}," or ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create New Table"]},"."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Optionally, type a table name."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Choose the method for importing the data."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Append"]}," (default)-Data import results are appended to the table."," ","If the table does not exist, it will be created."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Always Replace"]},"-Replaces the entire content of an existing table with the result output of the query. If the table does not exist, a new table is created."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Replace on New Data"]},"-Only replace the entire content of an existing table with the result output when there is new data."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Timestamp-based Partition Key"]}," column."," ","If you want to set a different partition key seed than the default key, you can specify the long or timestamp column as the partitioning time. As a default time column, it uses upload_time with the add_time filter."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Timezone"]}," for your data storage."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Under ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Schedule"]},", you can choose when and how often you want to run this query."]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"run-once","__idx":9},"children":["Run once"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Off"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Scheduling Timezone"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create & Run Now"]},"."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"repeat-regularly","__idx":10},"children":["Repeat Regularly"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["On"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Schedule"]},". The UI provides these four options: ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["@hourly"]},", ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["@daily"]}," and ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["@monthly"]}," or custom ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["cron"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You can also select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Delay Transfer"]}," and add a delay of execution time."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Scheduling Timezone"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create & Run Now"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After your transfer has run, you can see the results of your transfer in ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Data Workbench"]}," > ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Databases."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"import-with-sftp-via-treasure-workflow","__idx":11},"children":["Import with SFTP via Treasure Workflow"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Create and run a workflow"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"_export:\n  td:\n    database: workflow_sftp_v2\n    table: workflow_sftp_v2\n+import_from_sftp_v2:\n  td_load>: imports/seed.yml\n  database: ${td.database}\n  table: ${td.table}\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Modify the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["seed.yml"]}," file with your SFTP connection details for the import."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"in:\n  type: sftp_v2\n  host: HOST\n  port: <PORT, default is 22>\n  auth_method: key_pair\n  user: USER\n  secret_key_file:\n    content: |\n      -----BEGIN RSA PRIVATE KEY-----\n      Proc-Type: 4,ENCRYPTED\n      DEK-Info: AES-128-CBC...\n      ...\n      -----END RSA PRIVATE KEY-----\n  secret_key_passphrase: PASSPHRASE\n  user_directory_is_root: true\n  timeout: 600\n  path_prefix: /path/to/sample\n  parser:\n    skip_header_lines: 1\n    charset: UTF-8\n    newline: CRLF\n    type: csv\n    delimiter: ','\n    quote: '\"'\n    columns:\n    - {name: id, type: long}\n    - {name: account, type: long}\n    - {name: time, type: timestamp, format: \"%Y-%m-%d %H:%M:%S\"}\n    - {name: purchase, type: timestamp, format: \"%Y%m%d\"}\n    - {name: comment, type: string}\n    - {name: json_column, type: json}\nout:\n  mode: append\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Configuration Parameters"},"children":["Configuration Parameters"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Value"},"children":["Value"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["host:"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["(string, required)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["port:"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["(string, default: 22)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["auth_method:"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["(string ['password', 'key_pair'], required)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["user:"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["(string, required)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["password:"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["(string, default: null)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["secret_key_file:"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["(string, default: null). OpenSSH format is required."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["secret_key_passphrase:"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["(string, default: \"\")"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["user_directory_is_root:"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["(boolean, default: true)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["timeout: sftp connection timeout seconds"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["(integer, default: 600)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["path_prefix: Prefix of output paths"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["(string, required)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["incremental: enables incremental loading"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["(boolean, optional. default: true). If incremental loading is enabled, config diff for the next execution will include last_path parameter so that next execution skips files before the path. Otherwise, last_path will not be included."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["path_match_pattern:"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["regexp to match file paths. If a file path doesn't match with this pattern, the file will be skipped (regexp string, optional)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["total_file_count_limit:"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["maximum number of files to read (integer, optional)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["min_task_size (experimental):"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["minimum size of a task. If this is larger than 0, one task includes multiple input files. This is useful if too many number of tasks impacts performance of output or executor plugins badly. (integer, optional)"]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"import-with-sftp-via-the-cli-td-toolbelt","__idx":12},"children":["Import with SFTP via the CLI (TD Toolbelt)"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"install-td-toolbelt","__idx":13},"children":["Install TD Toolbelt"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Install the most current ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://toolbelt.treasuredata.com/"},"children":["Treasure Data Toolbelt"]},"."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td --version\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"create-seed-config-file-seedyml","__idx":14},"children":["Create Seed Config File (seed.yml)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Prepare ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["seed.yml,"]}," as shown in the following example, with your SFTP_v2 details. We support two authentication methods: Public / Private Key Pair and Password."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"public-and-private-key-pair-authentication","__idx":15},"children":["Public and Private Key Pair Authentication"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Create ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["seed.yml"]}," with the following content."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"in:\n  type: sftp_v2\n  host: HOST\n  port: <PORT, default is 22>\n  auth_method: key_pair\n  user: USER\n  secret_key_file:\n    content: |\n      -----BEGIN RSA PRIVATE KEY-----\n      Proc-Type: 4,ENCRYPTED\n      DEK-Info: AES-128-CBC...\n      ...\n      -----END RSA PRIVATE KEY-----\n  secret_key_passphrase: PASSPHRASE\n  user_directory_is_root: true\n  timeout: 600\n  path_prefix: /path/to/sample\nout:\n  mode: append\n  exec: {}\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["secret\\_key\\_file"]}," requires OpenSSH format."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"password-authentication","__idx":16},"children":["Password Authentication"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Create ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["seed.yml"]}," with the following content."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"in:\n  type: sftp_v2\n  host: HOST\n  port: <PORT, default is 22>\n  auth_method: password\n  user: USER\n  password: PASSWORD\n  user_directory_is_root: true\n  timeout: 600\n  path_prefix: /path/to/sample\nout:\n  mode: append\n  exec: {}\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can use the following special characters in the password: \"#$!*@\""]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The SFTP_v2 integration imports all files that match the specified prefix. path_prefix must point to a file or folder (e.g. path_prefix: ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample"]},"–> ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample/201501.csv.gz"]},", ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample/201502.csv.gz"]},", …, ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample/201505.csv.gz"]},")."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"guess-fields-generate-loadyml","__idx":17},"children":["Guess Fields (Generate load.yml)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Use ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["connector:guess"]},". This command automatically reads the source file and assesses (uses logic to guess) the file format."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td connector:guess seed.yml -o load.yml\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you open ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["load.yml"]},", you see the guessed file format definitions, including file formats, encodings, column names, and types. This example is trying to load CSV files."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"in:\n  type: sftp_v2\n  host: HOST\n  port: <PORT, default is 22>\n  auth_method: key_pair\n  user: USER\n  secret_key_file:\n    content: |\n      -----BEGIN RSA PRIVATE KEY-----\n      Proc-Type: 4,ENCRYPTED\n      DEK-Info: AES-128-CBC...\n      ...\n      -----END RSA PRIVATE KEY-----\n  secret_key_passphrase: PASSPHRASE\n  user_directory_is_root: true\n  timeout: 600\n  path_prefix: /path/to/sample\n  parser:\n    skip_header_lines: 1\n    charset: UTF-8\n    newline: CRLF\n    type: csv\n    delimiter: ','\n    quote: '\"'\n    columns:\n    - {name: id, type: long}\n    - {name: account, type: long}\n    - {name: time, type: timestamp, format: \"%Y-%m-%d %H:%M:%S\"}\n    - {name: purchase, type: timestamp, format: \"%Y%m%d\"}\n    - {name: comment, type: string}\n    - {name: json_column, type: json}\nout:\n  mode: append\n  exec: {}\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Then, you can preview how the system will parse the file by using the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["preview"]}," command."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"td connector:preview load.yml\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The guess command needs over 3 rows and 2 columns in source data file, because it guesses column definition using sample rows from source data."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If the system detects your column name or column type unexpectedly, modify ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["load.yml"]}," directly and preview again."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The integration supports parsing of “boolean”, “long”, “double”, “string”, and “timestamp” types."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You also must create a database and table prior to executing the data load job. Follow these steps:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td database:create td_sample_db\ntd table:create td_sample_db td_sample_table\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"execute-load-job","__idx":18},"children":["Execute Load Job"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Submit the load job. It may take a couple of hours, depending on the size of the data. Specify the Treasure Data database and table where the data should be stored."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["It’s also recommended to specify ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["--time-column"]}," option, because Treasure Data’s storage is partitioned by time (see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.treasuredata.com/smart/project-product-documentation/data-partitioning-in-treasure-data"},"children":["data partitioning"]},") If the option is not provided, the integration chooses the first ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["long"]}," or ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["timestamp"]}," column as the partitioning time. The type of the column specified by ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["--time-column"]}," must be either of ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["long"]}," and ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["timestamp"]}," type."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If your data doesn’t have a time column, you can add a time column by using ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["add_time"]}," filter option. For more details, see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.treasuredata.com/smart/project-product-documentation/add_time-filter-function"},"children":["add_time filter plugin"]},"."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:issue load.yml --database td_sample_db --table td_sample_table --time-column created_at\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The connector:issue command assumes that you have already created a *database(td_sample_db)*and a ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["table(td_sample_table)"]},". If the database or the table do not exist in TD, the connector:issue command fails. If this happens, ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.treasuredata.com/smart/project-product-documentation/creating-or-viewing-a-database"},"children":["create the database"]}," and ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.treasuredata.com/smart/project-product-documentation/creating-or-viewing-tables"},"children":["create a table"]}," manually, or use ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["--auto-create-table"]}," option with ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td connector:issue"]}," command to auto create the database and table:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:issue load.yml --database td_sample_db --table td_sample_table --time-column created_at --auto-create-table\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The integration does not sort records on server-side. To use time-based partitioning effectively, sort records beforehand."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you have a field called ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["time"]},", you don't have to specify the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["--time-column"]}," option."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:issue load.yml --database td_sample_db --table td_sample_table\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"scheduled-execution","__idx":19},"children":["Scheduled Execution"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can schedule periodic integration execution for incremental SFTP_v2 file import. We configure our scheduler carefully to ensure high availability. Using this feature means you no longer need a ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["cron"]},"daemon on your local data center."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For the scheduled import, the integration for SFTP_v2 imports all files that match with the specified prefix (e.g. path_prefix: ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample"]}," –> ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample/201501.csv.gz"]},", ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample/201502.csv.gz"]},", …, ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample/201505.csv.gz"]},") at first and remembers the last path (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample/201505.csv.gz"]},") for the next execution."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["On the second and on subsequent runs, it imports only files that come after the last path in alphabetical (lexicographic) order. (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample/201506.csv.gz"]},", …)"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"create-the-schedule","__idx":20},"children":["Create the Schedule"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["A new schedule can be created using the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td connector:create"]}," command. The following are required: the name of the schedule, the cron-style schedule, the database and table where the data will be stored, and the integration configuration file."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:create \\\ndaily_import \\\n\"10 0 * * *\" \\\ntd_sample_db \\\ntd_sample_table \\\nload.yml\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["It's also recommended to specify the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["--time-column"]}," option, because Treasure Data’s storage is partitioned by time."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:create \\\ndaily_import \\\n\"10 0 * * *\" \\\ntd_sample_db \\\ntd_sample_table \\\nload.yml \\\n--time-column created_at\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["cron"]}," parameter also accepts three special options: ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["@hourly"]},", ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["@daily"]}," and ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["@monthly"]},"."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["By default, the schedule is set up in UTC timezone. You can set the schedule in a timezone using -t or --timezone option. The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["--timezone"]}," option supports only extended timezone formats like 'Asia/Tokyo', 'America/Los_Angeles', etc. Timezone abbreviations like PST, and CST are *not* supported and may lead to unexpected schedules."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"list-the-schedules","__idx":21},"children":["List the Schedules"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can see the list of currently scheduled entries by running the command ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td connector:list"]},"."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"td connector:list\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"show-the-setting-and-schedule-history","__idx":22},"children":["Show the Setting and Schedule History"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td connector:show"]}," shows the execution setting of a schedule entry."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"td connector:show daily_import\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td connector:history"]}," shows the execution history of a schedule entry. To investigate the results of each individual run, use ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td job jobid"]},"."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:history daily_import\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"delete-the-schedule","__idx":23},"children":["Delete the Schedule"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td connector:delete"]}," will remove the schedule."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"td connector:delete daily_import\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"modes-for-out-plugin","__idx":24},"children":["Modes for out plugin"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can specify the file import mode in the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["out"]}," section of seed.yml."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"append-default","__idx":25},"children":["append (default)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["This is the default mode, and records are appended to the target table."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"in:\n  ...\nout:\n  mode: append\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"replace-in-td-01110-and-later","__idx":26},"children":["replace (In td 0.11.10 and later)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["This mode replaces data in the target table. Note that any manual schema changes made to the target table will remain intact with this mode."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"in:\n  ...\nout:\n  mode: replace\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"import-from-sftp-server-via-workflow","__idx":27},"children":["Import from SFTP Server via Workflow"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For sample workflows of importing files from your STFP server, view ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://github.com/treasure-data/treasure-boxes/tree/master/td_load/sftp"},"children":["Treasure Boxes"]},"."]}]},"headings":[{"value":"SFTP V2 Server Import Integration","id":"sftp-v2-server-import-integration","depth":1},{"value":"Prerequisites","id":"prerequisites","depth":2},{"value":"Limitations and Supported","id":"limitations-and-supported","depth":2},{"value":"Static IP Address of Treasure Data Integration","id":"static-ip-address-of-treasure-data-integration","depth":2},{"value":"Use the Treasure Console to Create Your Connection","id":"use-the-treasure-console-to-create-your-connection","depth":2},{"value":"Create a New Connection","id":"create-a-new-connection","depth":3},{"value":"Transfer Your Data to Treasure Data","id":"transfer-your-data-to-treasure-data","depth":2},{"value":"Data Preview","id":"data-preview","depth":3},{"value":"Data Placement","id":"data-placement","depth":3},{"value":"Run once","id":"run-once","depth":4},{"value":"Repeat Regularly","id":"repeat-regularly","depth":4},{"value":"Import with SFTP via Treasure Workflow","id":"import-with-sftp-via-treasure-workflow","depth":1},{"value":"Import with SFTP via the CLI (TD Toolbelt)","id":"import-with-sftp-via-the-cli-td-toolbelt","depth":1},{"value":"Install TD Toolbelt","id":"install-td-toolbelt","depth":2},{"value":"Create Seed Config File (seed.yml)","id":"create-seed-config-file-seedyml","depth":2},{"value":"Public and Private Key Pair Authentication","id":"public-and-private-key-pair-authentication","depth":3},{"value":"Password Authentication","id":"password-authentication","depth":3},{"value":"Guess Fields (Generate load.yml)","id":"guess-fields-generate-loadyml","depth":2},{"value":"Execute Load Job","id":"execute-load-job","depth":2},{"value":"Scheduled Execution","id":"scheduled-execution","depth":2},{"value":"Create the Schedule","id":"create-the-schedule","depth":3},{"value":"List the Schedules","id":"list-the-schedules","depth":3},{"value":"Show the Setting and Schedule History","id":"show-the-setting-and-schedule-history","depth":3},{"value":"Delete the Schedule","id":"delete-the-schedule","depth":3},{"value":"Modes for out plugin","id":"modes-for-out-plugin","depth":2},{"value":"append (default)","id":"append-default","depth":3},{"value":"replace (In td 0.11.10 and later)","id":"replace-in-td-01110-and-later","depth":3},{"value":"Import from SFTP Server via Workflow","id":"import-from-sftp-server-via-workflow","depth":1}],"frontmatter":{"seo":{"title":"SFTP V2 Server Import Integration"}},"lastModified":"2026-06-01T09:09:59.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/int/sftp_v2-server-import-integration","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}