{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":["admonition","partial"]},"redocly_category":"Integrations","type":"markdown"},"seo":{"title":"Fluentd Continuous Data Import Integration","description":"Treasure Data Product Documentation · Collect and Unify · Segment and Activate · Experiment and Analyze · Decisioning Automate with AI Scale and Trust.","siteUrl":"https://docs.treasuredata.com","lang":"en-US","llmstxt":{"hide":false,"sections":[{"title":"Table of contents","includeFiles":["**/*"],"excludeFiles":[]}],"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"fluentd-continuous-data-import-integration","__idx":0},"children":["Fluentd Continuous Data Import Integration"]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["td-agent was discontinued in December 2023 and has been replaced by ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["fluent-package"]},". The fluent-package is the official successor maintained by the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://www.cncf.io/projects/"},"children":["Cloud Native Computing Foundation"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can continuously import the data into Treasure Data using Fluentd."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"http://fluentd.org/"},"children":["Fluentd"]}," is a daemon written in Ruby that collects logs as JSON streams, buffers the logs, and sends the logs to other systems such as MySQL, MongoDB, or even other instances of Fluentd."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"prerequisites","__idx":1},"children":["Prerequisites"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Basic knowledge of ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"http://fluentd.org/"},"children":["Fluentd"]},", and its installation."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"installing-fluentd","__idx":2},"children":["Installing Fluentd"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"fluentd-fluent-package-install-options","__idx":3},"children":["Fluentd (fluent-package) Install Options"]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["td-agent was discontinued in December 2023 and has been replaced by ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["fluent-package"]},". The fluent-package is the official successor maintained by the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://www.cncf.io/projects/"},"children":["Cloud Native Computing Foundation"]},". For migration guidance from td-agent, see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.fluentd.org/installation/install-fluent-package"},"children":["Fluentd Installation Guide"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To install ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["fluent-package"]},", run one of the following commands based on your environment."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"rhelcentosrocky-linux","__idx":4},"children":["RHEL/CentOS/Rocky Linux"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# fluent-package 6 LTS (recommended)\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-redhat-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"ubuntu","__idx":5},"children":["Ubuntu"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Ubuntu 24.04 Noble - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-ubuntu-noble-fluent-package6-lts.sh | sh\n\n# Ubuntu 22.04 Jammy - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-ubuntu-jammy-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"debian","__idx":6},"children":["Debian"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Debian Bookworm - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-debian-bookworm-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"amazon-linux","__idx":7},"children":["Amazon Linux"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Amazon Linux 2023 - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-amazon2023-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"windows","__idx":8},"children":["Windows"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Download the MSI installer from:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://fluentd.cdn.cncf.io/lts/6/windows/index.html"},"children":["fluent-package 6 LTS for Windows"]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After installation:"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Edit the configuration file at ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["C:/opt/fluent/etc/fluent/fluentd.conf"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Start the service using ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["net start fluentdwinsvc"]}," or via Services administrative tool"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"macos","__idx":9},"children":["macOS"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["fluent-package for macOS is planned to be available via Homebrew. For current installation options, see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.fluentd.org/installation"},"children":["Fluentd Installation Guide"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"starting-the-service","__idx":10},"children":["Starting the Service"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After installation, start and verify the Fluentd service."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"linux","__idx":11},"children":["Linux"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"sudo systemctl start fluentd.service\nsudo systemctl status fluentd.service\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The configuration file is located at ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/etc/fluent/fluentd.conf"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"windows-1","__idx":12},"children":["Windows"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"cmd","header":{"controls":{"copy":{}}},"source":"net start fluentdwinsvc\n","lang":"cmd"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The configuration file is located at ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["C:\\opt\\fluent\\etc\\fluent\\fluentd.conf"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"macos-gem-installation","__idx":13},"children":["macOS (gem installation)"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"fluentd -c /path/to/fluentd.conf\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For more details, see the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.fluentd.org/"},"children":["Fluentd Documentation"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"installing-the-fluent-plugin-td","__idx":14},"children":["Installing the fluent-plugin-td"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you're using Fluentd installed via Ruby gems instead of fluent-package, install the fluentd and fluent-plugin-td gems as follows:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"gem install fluentd\ngem install fluent-plugin-td\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"configuration","__idx":15},"children":["Configuration"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Add the following lines to your configuration file (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/etc/fluent/fluentd.conf"]}," for fluent-package). The fluent-plugin-td contains the tdlog output type, which is used to upload data into Treasure Data."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"conf","header":{"controls":{"copy":{}}},"source":"# HTTP input\n<source>\n  @type http\n  port 8888\n</source>\n\n# TCP input\n<source>\n  @type forward\n  port 24224\n</source>\n\n# Treasure Data output\n<match td.*.*>\n  @type tdlog\n  endpoint api.treasuredata.com\n  apikey YOUR_API_KEY\n  auto_create_table\n  use_ssl true\n  <buffer>\n    @type file\n    path /var/log/fluent/buffer/td\n  </buffer>\n</match>\n","lang":"conf"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The API key is a secret key to authenticate your account. You can get your API key from your profile in Treasure Console."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"confirm-data-upload","__idx":16},"children":["Confirm Data Upload"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Start the Fluentd service (if using fluent-package):"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"sudo systemctl start fluentd.service\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Or start Fluentd manually (if using gem installation):"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"fluentd --config fluentd.conf\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Add logs in JSON using HTTP."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"curl -X POST -d 'json={\"action\":\"login\",\"user\":2}' \\\n  http://localhost:8888/td.testdb.www_access\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Fluentd continuously uploads logs, sending data every minute. Sending a SIGUSR1 signal forces Fluentd to flush the buffered logs into the cloud immediately."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"kill -USR1 $(cat /var/run/fluent/fluentd.pid)\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Issue the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["td tables"]}," command to confirm that your data imported successfully."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"$ td tables\n+------------+------------+------+-----------+\n| Database   | Table      | Type | Count     |\n+------------+------------+------+-----------+\n| testdb     | www_access | log  | 1         |\n+------------+------------+------+-----------+\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If you run into any issues, Fluentd's log (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/var/log/fluent/fluentd.log"]},") is a good place to start troubleshooting."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"other-resources","__idx":17},"children":["Other Resources"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.fluentd.org/"},"children":["Fluentd Documentation"]}]}]}]},"headings":[{"value":"Fluentd Continuous Data Import Integration","id":"fluentd-continuous-data-import-integration","depth":1},{"value":"Prerequisites","id":"prerequisites","depth":2},{"value":"Installing Fluentd","id":"installing-fluentd","depth":2},{"value":"Fluentd (fluent-package) Install Options","id":"fluentd-fluent-package-install-options","depth":2},{"value":"RHEL/CentOS/Rocky Linux","id":"rhelcentosrocky-linux","depth":3},{"value":"Ubuntu","id":"ubuntu","depth":3},{"value":"Debian","id":"debian","depth":3},{"value":"Amazon Linux","id":"amazon-linux","depth":3},{"value":"Windows","id":"windows","depth":3},{"value":"macOS","id":"macos","depth":3},{"value":"Starting the Service","id":"starting-the-service","depth":2},{"value":"Linux","id":"linux","depth":3},{"value":"Windows","id":"windows-1","depth":3},{"value":"macOS (gem installation)","id":"macos-gem-installation","depth":3},{"value":"Installing the fluent-plugin-td","id":"installing-the-fluent-plugin-td","depth":2},{"value":"Configuration","id":"configuration","depth":2},{"value":"Confirm Data Upload","id":"confirm-data-upload","depth":2},{"value":"Other Resources","id":"other-resources","depth":2}],"frontmatter":{"seo":{"title":"Fluentd Continuous Data Import Integration"}},"lastModified":"2026-06-01T09:09:59.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/int/fluentd-continuous-data-import-integration","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}