{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":["admonition"]},"redocly_category":"Integrations","type":"markdown"},"seo":{"title":"SFTP Server Import Integration","description":"Treasure Data Product Documentation · Collect and Unify · Segment and Activate · Experiment and Analyze · Decisioning Automate with AI Scale and Trust.","siteUrl":"https://docs.treasuredata.com","lang":"en-US","llmstxt":{"hide":false,"sections":[{"title":"Table of contents","includeFiles":["**/*"],"excludeFiles":[]}],"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"sftp-server-import-integration","__idx":0},"children":["SFTP Server Import Integration"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The Data Connector for SFTP enables you to import files stored on your SFTP server to Treasure Data."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For SFTP_V2 Server Import Integration, see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/int/sftp_v2-server-import-integration"},"children":["SFTP_V2 Server Import Integration"]},"."]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"warning"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The SFTP Server Import integration will no longer be subject to active development or maintenance. Treasure Data recommends using the SFTP_V2 Server Import Integration integration."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"prerequisites","__idx":1},"children":["Prerequisites"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Basic knowledge of Treasure Data."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Requires that your private key is in a version format earlier than OpenSSH 7.8. If you use OpenSSH 7.8 or later, use the '-m PEM' option to generate a private key."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Before using this connector, determine valid protocols for your environment."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["If you intend to use SFTP, you can use this integration for SFTP."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["If you intend to use FTP or FTPS, try connecting with ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/int/ftp-server-import-integration"},"children":["FTP Import Integration"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Check your accepted IP range and port if you are using a firewall. Sometimes, server administrators change the default port number from TCP 22 for security reasons."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["“PuTTY” and other formats are not supported."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After installation and configuration, review the Job logs for warnings and errors that can provide information about the success of your import."]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"limitations","__idx":2},"children":["Limitations"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Support only the STORED and DEFLATE compression methods."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Multi-part gzip file may not work."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["The connection timeout setting doesn't function properly in v.1 so the import job could run for a long time instead of timing out."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"static-ip-address-of-treasure-data-integration","__idx":3},"children":["Static IP Address of Treasure Data Integration"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If your security policy requires IP whitelisting, you must add Treasure Data's IP addresses to your allowlist to ensure a successful connection."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Please find the complete list of static IP addresses, organized by region, at the following ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/apis/endpoints/ip-addresses-integrations-result-workers"},"children":["document"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"import-from-sftp-server-via-treasure-console","__idx":4},"children":["Import from SFTP server via Treasure Console"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can use Treasure Console to create your data connector."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"create-a-new-connection","__idx":5},"children":["Create a New Connection"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["When you configure a data connection, you need to provide authentication to access the integration. In Treasure Data, you can configure the authentication and then specify the source information."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Open Treasure Console."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Navigate to ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Integrations Hub"]}," > ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Catalog"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Search and select SFTP."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image-20200724-000304.6361afa5561c2c0d9312bb7a954d5e289147935a95ae7d0be38293a93048fa2b.4ed12e38.png","alt":""},"children":[]}," ","4. Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create,"]}," and the following dialog opens."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image-20200724-000342.5603aa7231805bfc19a976c40f9607387ebfa4a73f8571dee4a8705d68d41819.4ed12e38.png","alt":""},"children":[]}," ","5. Enter the required credentials for your remote SFTP instance."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Host"]},": The host information of the remote SFTP instance, for example, an IP address."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Port"]},": The connection port on the remote SFTP instance. The default port number is 22."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["User"]},": The user name used to connect to the remote FTP instance."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Authentication mode"]},": The way you choose to authenticate with your SFTP server."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Secret key file"]},": Required if ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["public / private key pair"]}," is selected as Authentication Mode. (DSA, RSA, and ECDSA are supported.)"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Passphrase for secret key file"]},": (Optional) If required, provide a passphrase for the provided secret file."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Retry limit"]},": Number of times to retry a failed connection (default 10)."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Timeout"]},": Connection timeout in seconds (default 600)."]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":6},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Continue"]},". Provide a name for your connection."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image-20200228-194345.f6f2c6392a4e8c4a61c5fc06e3fce6fa41fee5fb3d2988a6c90eb320e7bb594a.4ed12e38.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":7},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["If you would like to share this connection with other users in your organization, check the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Share with others"]}," checkbox. If this box is unchecked, this connection is visible only to you."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Done."]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"transfer-data-into-treasure-data","__idx":6},"children":["Transfer Data into Treasure Data"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To get the data from your SFTP server into Treasure Data, you can set up an ad hoc one-time or recurring transfer at a regular intervals. After creating the authenticated connection, you are automatically taken to Authentications."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Search for the connection you created."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["New Source"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create a Connection"]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":3},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Type a name for your ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Source"]}," in the Data Transfer field**.**"]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image-20200724-001504.e67e581aecf36d4d38b39c1ee808d26645db8f81e5048fbf1fedb59a2932f48c.4ed12e38.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":4},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Click ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next"]},"."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"identify-asource-table","__idx":7},"children":["Identify a Source Table"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["The Source dialog opens."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image-20200724-001612.6a7cb19e8231f3043d9fa828e8307373d88ac9422b99b4e3c8e130268f48ac7b.4ed12e38.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":2},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Edit the following parameters."]}]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Parameters"]}]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Description"]}]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["User directory root"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Missing"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Path prefix"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The prefix of target files (string, required)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Path match pattern"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Type a regular expression to query file paths. The file is skipped if a file path does not match the specified pattern. For example, if you specify the pattern  .csv$ # , then a file is skipped if its path does not match the pattern."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Incremental"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Enables incremental loading (boolean, optional. default: true. If incremental loading is enabled, the config diff for the next execution will include ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["last_path"]}," parameter so that the next execution skips files before the path. Otherwise, ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["last_path"]}," is not included."]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Start after path"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Only paths lexicographically greater than this will be imported."]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"_","__idx":8},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Data Settings"]}]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next"]},". The Data Settings page opens."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["(Optional) Edit the data settings."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image-20200724-002019.fc04f773c72f5db7b70b69d46395a1a5b2747d60f9c74cbdbdca37efa6030d98.4ed12e38.png","alt":""},"children":[]}," ",{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image-2023-08-17.8bf4488f6a24c023564fb641a83d36fe324d065e65988518165f784c5ebd1520.4ed12e38.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"filters","__idx":9},"children":["Filters"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Filters are available in the Create Source or Edit Source import settings for your S3, FTP, or SFTP connectors."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Import Integration Filters enable you to modify your imported data after you have completed ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.treasuredata.com/smart/project-product-documentation/editing-data-settings"},"children":["Editing Data Settings"]}," for your import."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To apply import integration filters:"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next"]}," in Data Settings.The Filters dialog opens."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select the filter option you want to add.",{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image-20200609-201955.eed6c6da800ba40d1d98b92e767d9a8f7500cad8a9d4079121190b7d34c23294.c7246827.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Add Filter."]}," The parameter dialog for that filter opens."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Edit the parameters. For information on each filter type, see one of the following:"]}]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Retaining Columns Filter"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Adding Columns Filter"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Dropping Columns Filter"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Expanding JSON Filter"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Digesting Filter"]}]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":5},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Optionally, to add another filter of the same type, select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Add"]}," within the specific column filter dialog."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Optionally, to add another filter of a different type, select the filter option from the list and repeat the same steps."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["After you have added the filters you want, select **Next.**The Data Preview dialog opens."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"data-preview","__idx":10},"children":["Data Preview"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can see a ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/integration-hub/batch/import/previewing-your-source-data"},"children":["preview"]}," of your data before running the import by selecting Generate Preview. Data preview is optional and you can safely skip to the next page of the dialog if you choose to."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next"]},". The Data Preview page opens."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["If you want to preview your data, select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Generate Preview"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Verify the data."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"data-placement","__idx":11},"children":["Data Placement"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For data placement, select the target database and table where you want your data placed and indicate how often the import should run."]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Next."]}," Under Storage, you will create a new or select an existing database and create a new or select an existing table for where you want to place the imported data."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select a ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Database"]}," > ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Select an existing"]}," or ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create New Database"]},"."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Optionally, type a database name."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select a ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Table"]},"> ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Select an existing"]}," or ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create New Table"]},"."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Optionally, type a table name."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Choose the method for importing the data."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Append"]}," (default)-Data import results are appended to the table."," ","If the table does not exist, it will be created."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Always Replace"]},"-Replaces the entire content of an existing table with the result output of the query. If the table does not exist, a new table is created."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Replace on New Data"]},"-Only replace the entire content of an existing table with the result output when there is new data."]}]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Timestamp-based Partition Key"]}," column."," ","If you want to set a different partition key seed than the default key, you can specify the long or timestamp column as the partitioning time. As a default time column, it uses upload_time with the add_time filter."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Select the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Timezone"]}," for your data storage."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Under ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Schedule"]},", you can choose when and how often you want to run this query."]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"run-once","__idx":12},"children":["Run once"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Off"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Scheduling Timezone"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create & Run Now"]},"."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"repeat-regularly","__idx":13},"children":["Repeat Regularly"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["On"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Schedule"]},". The UI provides these four options: ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["@hourly"]},", ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["@daily"]}," and ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["@monthly"]}," or custom ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["cron"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["You can also select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Delay Transfer"]}," and add a delay of execution time."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Scheduling Timezone"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Select ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Create & Run Now"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After your transfer has run, you can see the results of your transfer in ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Data Workbench"]}," > ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Databases."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"import-from-sftp-server-via-cli-toolbelt","__idx":14},"children":["Import from SFTP server via CLI (Toolbelt)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can also use the SFTP data connector from the command line interface. The following instructions show you how to import data using the CLI."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"install-td-command-v0119-or-later","__idx":15},"children":["Install ‘td’ command v0.11.9 or later"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Install the most current ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://toolbelt.treasuredata.com/"},"children":["Treasure Data Toolbelt"]},"."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td --version\n0.11.10\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"create-seed-config-file-seedyml","__idx":16},"children":["Create Seed Config File (seed.yml)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Prepare ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["seed.yml"]}," as shown in the following example, with your SFTP details. We support two authentication methods: Public / Private Key Pair, and Password."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"public-and-private-key-pair-authentication","__idx":17},"children":["Public and Private Key Pair Authentication"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Create ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["seed.yml"]}," with the following content."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"in:\n  type: sftp\n  host: HOST\n  port: <PORT, default is 22>\n  user: USER\n  secret_key_file:\n content: |\n-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC...\n...\n-----END RSA PRIVATE KEY-----\n  secret_key_passphrase: PASSPHRASE\n  user_directory_is_root: true\n  timeout: 600\n  path_prefix: /path/to/sample\nout:\n  mode: append\n  exec: {}\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["secret_key_file"]}," requires OpenSSH format."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"password-authentication","__idx":18},"children":["Password Authentication"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Create ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["seed.yml"]}," with the following content."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"in:\n  type: sftp\n  host: HOST\n  port: <PORT, default is 22>\n  auth_method: password\n  user: USER\n  password: PASSWORD\n  user_directory_is_root: true\n  timeout: 600\n  path_prefix: /path/to/sample\nout:\n  mode: append\n  exec: {}\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can use the following special characters in the password: \"#$!*@\""]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you are using the proxy, add the additional information as shown:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"in:\n  type: sftp\n  host: HOST\n  port: <PORT, default is 22>\n  ....\n  proxy:\n    type: http\n    host: PROXY_HOST\n    port: PROXY_PORT\n    user: PROXY_USER\n    password: PROXY_PASSWORD\n    command: SOMETHING COMMAND IF NEEDED\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The Data Connector for SFTP imports all files that match the specified prefix (e.g. path_prefix: ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_"]}," –> ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201501.csv.gz"]},", ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201502.csv.gz"]},", …, ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201505.csv.gz"]},")."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"guess-fields-generate-loadyml","__idx":19},"children":["Guess Fields (Generate load.yml)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Use ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["connector:guess"]},". This command automatically reads the source file, and assesses (uses logic to guess) the file format."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td connector:guess seed.yml -o load.yml\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you open ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["load.yml"]},", you’ll see the guessed file format definitions including file formats, encodings, column names, and types. This example is trying to load CSV files."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"in:\n  type: sftp\n  host: HOST\n  port: <PORT, default is 22>\n  user: USER\n  secret_key_file:\n content: |\n-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC...\n...\n-----END RSA PRIVATE KEY-----\n  secret_key_passphrase: PASSPHRASE\n  user_directory_is_root: true\n  timeout: 600\n  path_prefix: /path/to/sample\n  parser:\n skip_header_lines: 1\n charset: UTF-8\n newline: CRLF\n type: csv\n delimiter: ','\n quote: '\"'\n columns:\n - {name: id, type: long}\n - {name: account, type: long}\n - {name: time, type: timestamp, format: \"%Y-%m-%d %H:%M:%S\"}\n - {name: purchase, type: timestamp, format: \"%Y%m%d\"}\n - {name: comment, type: string}\n - {name: json_column, type: json}\nout:\n  mode: append\n  exec: {}\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Then, you can preview how the system will parse the file by using the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["preview"]}," command."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td connector:preview load.yml\n+-------+---------+----------+---------------------+\n| id | company | customer | created_at |\n+-------+---------+----------+---------------------+\n| 11200 | AA Inc. | David | 2015-03-31 06:12:37 |\n| 20313 | BB Imc. |Tom | 2015-04-01 01:00:07 |\n| 32132 | CC Inc. | Fernando | 2015-04-01 10:33:41 |\n| 40133 | DD Inc. | Cesar | 2015-04-02 05:12:32 |\n| 93133 | EE Inc. |  Jake | 2015-04-02 14:11:13 |\n+-------+---------+----------+---------------------+\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The guess command needs over 3 rows and 2 columns in the source data file because it guesses column definition using sample rows from source data."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If the system detects your column name or column type unexpectedly, modify ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["load.yml"]}," directly and preview again."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Currently, the Data Connector supports parsing of “boolean”, “long”, “double”, “string”, and “timestamp” types."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You also must create a local database and table prior to executing the data load job. Follow these steps:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td database:create td_sample_db\n$ td table:create td_sample_db td_sample_table\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"execute-load-job","__idx":20},"children":["Execute Load Job"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Submit the load job. It may take a couple of hours depending on the size of the data. Specify the Treasure Data database and table where the data should be stored."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["It’s also recommended to specify ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["--time-column"]}," option, because Treasure Data’s storage is partitioned by time. If the option is not provided, the data connector chooses the first ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["long"]}," or ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["timestamp"]}," column as the partitioning time. The type of the column specified by ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["--time-column"]}," must be either of ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["long"]}," and ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["timestamp"]}," type."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If your data doesn’t have a time column you can add a time column by using ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["add_time"]}," filter option. For more details see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.treasuredata.com/smart/project-product-documentation/add_time-filter-function"},"children":["add_time filter plugin"]},"."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td connector:issue load.yml --database td_sample_db --table td_sample_table --time-column created_at\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The connector:issue command assumes that you have already created a *database(td_sample_db)*and a ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["table(td_sample_table)"]},". If the database or the table do not exist in TD, the connector:issue command fails, so create the database and table ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.treasuredata.com/smart/project-product-documentation/data-management"},"children":["manually"]}," or use ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["--auto-create-table"]}," option with ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td connector:issue"]}," command to auto create the database and table:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td connector:issue load.yml --database td_sample_db --table td_sample_table --time-column created_at --auto-create-table\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The Data Connector does not sort records on server-side. To use time-based partitioning effectively, sort records in files beforehand."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you have a field called ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["time"]},", you don't have to specify the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["--time-column"]}," option."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td connector:issue load.yml --database td_sample_db --table td_sample_table\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"scheduled-execution","__idx":21},"children":["Scheduled Execution"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can schedule periodic Data Connector execution for incremental SFTP file import. We configure our scheduler carefully to ensure high availability. By using this feature, you no longer need a ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["cron"]},"daemon on your local data center."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For the scheduled import, the Data Connector for SFTP imports all files that match with the specified prefix (e.g. path_prefix: ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_"]}," –> ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201501.csv.gz"]},", ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201502.csv.gz"]},", …, ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201505.csv.gz"]},") at first and remembers the last path (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201505.csv.gz"]},") for the next execution."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["On the second and on subsequent runs, it imports only files that comes after the last path in alphabetical (lexicographic) order. (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["path/to/sample_201506.csv.gz"]},", …)"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"create-the-schedule","__idx":22},"children":["Create the Schedule"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["A new schedule can be created using the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td connector:create"]}," command. The following are required: the name of the schedule, the cron-style schedule, the database and table where the data will be stored, and the Data Connector configuration file."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td connector:create\n daily_import\n \"10 0 * * *\"\n td_sample_db\n td_sample_table\n load.yml\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["It’s also recommended to specify the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["--time-column"]}," option because Treasure Data’s storage is partitioned by time."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td connector:create\n daily_import\n \"10 0 * * *\"\n td_sample_db\n td_sample_table\n load.yml\n --time-column created_at\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["cron"]}," parameter also accepts three special options: ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["@hourly"]},", ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["@daily"]}," and ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["@monthly"]},"."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["By default, the schedule is set up in UTC timezone. You can set the schedule in a timezone using -t or --timezone option. The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["--timezone"]}," option supports only extended timezone formats like 'Asia/Tokyo', 'America/Los_Angeles' etc. Timezone abbreviations like PST, CST are ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["not"]}," supported and may lead to unexpected schedules."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"list-the-schedules","__idx":23},"children":["List the Schedules"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can see the list of currently scheduled entries by running the command ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td connector:list"]},"."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"$ td connector:list\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"show-the-setting-and-schedule-history","__idx":24},"children":["Show the Setting and Schedule History"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td connector:show"]}," shows the execution setting of a schedule entry."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td connector:show daily_import\nName  : daily_import\nCron  : 10 0 * * *\nTimezone : UTC\nDelay : 0\nDatabase : td_sample_db\nTable : td_sample_table\nConfig\n---\nin:\n  type: sftp\n  host: HOST\n  port: <PORT, default is 22>\n  auth_method: password\n  user: USER\n  password: PASSWORD\n  path_prefix: /sftp/file/path/prefix\n  parser:\n charset: UTF-8\n ...\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td connector:history"]}," shows the execution history of a schedule entry. To investigate the results of each individual run, use ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td job jobid"]},"."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"% td connector:history daily_import\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"import-from-sftp-server-via-workflow","__idx":25},"children":["Import from SFTP Server via Workflow"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For sample workflows of importing files from your STFP server, view ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://github.com/treasure-data/treasure-boxes/tree/master/td_load/sftp"},"children":["Treasure Boxes"]},"."]}]},"headings":[{"value":"SFTP Server Import Integration","id":"sftp-server-import-integration","depth":1},{"value":"Prerequisites","id":"prerequisites","depth":2},{"value":"Limitations","id":"limitations","depth":2},{"value":"Static IP Address of Treasure Data Integration","id":"static-ip-address-of-treasure-data-integration","depth":2},{"value":"Import from SFTP server via Treasure Console","id":"import-from-sftp-server-via-treasure-console","depth":2},{"value":"Create a New Connection","id":"create-a-new-connection","depth":3},{"value":"Transfer Data into Treasure Data","id":"transfer-data-into-treasure-data","depth":3},{"value":"Identify a Source Table","id":"identify-asource-table","depth":3},{"value":"","id":"_","depth":3},{"value":"Filters","id":"filters","depth":3},{"value":"Data Preview","id":"data-preview","depth":3},{"value":"Data Placement","id":"data-placement","depth":3},{"value":"Run once","id":"run-once","depth":4},{"value":"Repeat Regularly","id":"repeat-regularly","depth":4},{"value":"Import from SFTP server via CLI (Toolbelt)","id":"import-from-sftp-server-via-cli-toolbelt","depth":2},{"value":"Install ‘td’ command v0.11.9 or later","id":"install-td-command-v0119-or-later","depth":3},{"value":"Create Seed Config File (seed.yml)","id":"create-seed-config-file-seedyml","depth":3},{"value":"Public and Private Key Pair Authentication","id":"public-and-private-key-pair-authentication","depth":3},{"value":"Password Authentication","id":"password-authentication","depth":3},{"value":"Guess Fields (Generate load.yml)","id":"guess-fields-generate-loadyml","depth":3},{"value":"Execute Load Job","id":"execute-load-job","depth":3},{"value":"Scheduled Execution","id":"scheduled-execution","depth":3},{"value":"Create the Schedule","id":"create-the-schedule","depth":3},{"value":"List the Schedules","id":"list-the-schedules","depth":3},{"value":"Show the Setting and Schedule History","id":"show-the-setting-and-schedule-history","depth":3},{"value":"Import from SFTP Server via Workflow","id":"import-from-sftp-server-via-workflow","depth":2}],"frontmatter":{"seo":{"title":"SFTP Server Import Integration"}},"lastModified":"2026-06-01T09:09:59.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/int/sftp-server-import-integration","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}