{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":["admonition","partial"]},"redocly_category":"Integrations","type":"markdown"},"seo":{"title":"Node.js Apps Import Integration","description":"Treasure Data Product Documentation · Collect and Unify · Segment and Activate · Experiment and Analyze · Decisioning Automate with AI Scale and Trust.","siteUrl":"https://docs.treasuredata.com","lang":"en-US","llmstxt":{"hide":false,"sections":[{"title":"Table of contents","includeFiles":["**/*"],"excludeFiles":[]}],"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"nodejs-apps-import-integration","__idx":0},"children":["Node.js Apps Import Integration"]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["td-agent was discontinued in December 2023 and has been replaced by ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["fluent-package"]},". The fluent-package is the official successor maintained by the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://www.cncf.io/projects/"},"children":["Cloud Native Computing Foundation"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Treasure Data provides ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/integration-hub/streaming/td-agent/about-treasure-data-s-server-side-agent"},"children":["Fluentd"]}," to collect server-side logs and events and to seamlessly import the data from Node.js applications."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The fluent-logger-node library enables Node.js applications to post records to their local Fluentd."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Fluentd, in turn, uploads the data to the cloud every 5 minutes. Because the daemon runs on a local node, the logging latency is negligible."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"prerequisites","__idx":1},"children":["Prerequisites"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Basic knowledge of Node.js and npm."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Basic knowledge of Treasure Data."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Node.js 22 or higher (for local testing)."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"installing-fluentd","__idx":2},"children":["Installing Fluentd"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Install Fluentd (fluent-package) on your application servers. Fluentd sits within your application servers, focusing on uploading application logs to the cloud."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image2020-12-2_14-6-46.1754f4e3154420d54cc976b16d454ef773c83ad5fc4b8580d99a87302c614932.7c05c672.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://github.com/fluent/fluent-logger-node"},"children":["fluent-logger-node"]}," library enables Node.js applications to post records to their local Fluentd. Fluentd, in turn, uploads the data to the cloud every 5 minutes. Because the daemon runs on a local node, the logging latency is negligible."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"fluentd-fluent-package-install-options","__idx":3},"children":["Fluentd (fluent-package) Install Options"]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["td-agent was discontinued in December 2023 and has been replaced by ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["fluent-package"]},". The fluent-package is the official successor maintained by the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://www.cncf.io/projects/"},"children":["Cloud Native Computing Foundation"]},". For migration guidance from td-agent, see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.fluentd.org/installation/install-fluent-package"},"children":["Fluentd Installation Guide"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To install ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["fluent-package"]},", run one of the following commands based on your environment."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"rhelcentosrocky-linux","__idx":4},"children":["RHEL/CentOS/Rocky Linux"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# fluent-package 6 LTS (recommended)\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-redhat-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"ubuntu","__idx":5},"children":["Ubuntu"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Ubuntu 24.04 Noble - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-ubuntu-noble-fluent-package6-lts.sh | sh\n\n# Ubuntu 22.04 Jammy - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-ubuntu-jammy-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"debian","__idx":6},"children":["Debian"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Debian Bookworm - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-debian-bookworm-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"amazon-linux","__idx":7},"children":["Amazon Linux"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Amazon Linux 2023 - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-amazon2023-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"windows","__idx":8},"children":["Windows"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Download the MSI installer from:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://fluentd.cdn.cncf.io/lts/6/windows/index.html"},"children":["fluent-package 6 LTS for Windows"]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After installation:"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Edit the configuration file at ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["C:/opt/fluent/etc/fluent/fluentd.conf"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Start the service using ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["net start fluentdwinsvc"]}," or via Services administrative tool"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"macos","__idx":9},"children":["macOS"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["fluent-package for macOS is planned to be available via Homebrew. For current installation options, see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.fluentd.org/installation"},"children":["Fluentd Installation Guide"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"starting-the-service","__idx":10},"children":["Starting the Service"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After installation, start and verify the Fluentd service."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"linux","__idx":11},"children":["Linux"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"sudo systemctl start fluentd.service\nsudo systemctl status fluentd.service\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The configuration file is located at ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/etc/fluent/fluentd.conf"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"windows-1","__idx":12},"children":["Windows"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"cmd","header":{"controls":{"copy":{}}},"source":"net start fluentdwinsvc\n","lang":"cmd"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The configuration file is located at ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["C:\\opt\\fluent\\etc\\fluent\\fluentd.conf"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"macos-gem-installation","__idx":13},"children":["macOS (gem installation)"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"fluentd -c /path/to/fluentd.conf\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For more details, see the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.fluentd.org/"},"children":["Fluentd Documentation"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"modifying-fluentdconf","__idx":14},"children":["Modifying fluentd.conf"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Specify your API key by setting the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["apikey"]}," option in your ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/etc/fluent/fluentd.conf"]}," file (for fluent-package). You can view your API key from your profile in Treasure Console."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"conf","header":{"controls":{"copy":{}}},"source":"# Treasure Data Input and Output\n<source>\n  @type forward\n  port 24224\n</source>\n\n<match td.*.*>\n  @type tdlog\n  endpoint api.treasuredata.com\n  apikey YOUR_API_KEY\n  auto_create_table\n  use_ssl true\n  <buffer>\n    @type file\n    path /var/log/fluent/buffer/td\n  </buffer>\n</match>\n","lang":"conf"},"children":[]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["YOUR_API_KEY"]}," should be your actual apikey string. You can retrieve your API key from your profiles in Treasure Console. Using a ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/my-settings/getting-your-api-keys"},"children":["write-only API key"]}," is recommended."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Restart the Fluentd service when the following lines are in place."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Linux\nsudo systemctl restart fluentd.service\n\n# macOS (gem installation)\n# Restart the Fluentd process manually\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Fluentd now accepts data via port 24224, buffers the data (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/var/log/fluent/buffer/td"]},"), and automatically uploads the data into the cloud."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"using-fluent-logger-node","__idx":15},"children":["Using fluent-logger-node"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"obtaining-the-most-recent-version","__idx":16},"children":["Obtaining the Most Recent Version"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Obtain the most recent version of ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://github.com/fluent/fluent-logger-node"},"children":["fluent-logger-node"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"a-sample-application","__idx":17},"children":["A Sample Application"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["A sample ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"http://expressjs.com/"},"children":["Express"]}," app using fluent-logger-node is as follows:"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"packagejson","__idx":18},"children":["package.json"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"{\n  \"name\": \"node-example\",\n  \"version\": \"0.0.1\",\n  \"dependencies\": {\n    \"express\": \"2.5.9\",\n    \"fluent-logger\": \"0.1.0\"\n  }\n}\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Now use ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["npm"]}," to install your dependencies locally:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ npm install\nfluent-logger@0.1.0 ./node_modules/fluent-logger\nexpress@2.5.9 ./node_modules/express\n|-- qs@0.4.2\n|-- mime@1.2.4\n|-- mkdirp@0.3.0\n|-- connect@1.8.6 (formidable@1.0.9)\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"webjs","__idx":19},"children":["web.js"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["This is the simplest web app."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"var express = require('express');\nvar app = express.createServer(express.logger());\n\nvar logger = require('fluent-logger');\nlogger.configure('td.test_db', {host: 'localhost', port: 24224});\n\napp.get('/', function(request, response) {\n  logger.emit('follow', {from: 'userA', to: 'userB'});\n  response.send('Hello World!');\n});\nvar port = process.env.PORT || 3000;\napp.listen(port, function() {\n  console.log(\"Listening on \" + port);\n});\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"confirming-data-import","__idx":20},"children":["Confirming Data Import"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Execute the app and go to ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["http://localhost:3000/"]}," in your browser."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"$ node web.js\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Sending a SIGUSR1 signal flushes Fluentd's buffer. Upload starts immediately."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Linux\n$ kill -USR1 $(cat /var/run/fluent/fluentd.pid)\n\n# macOS (gem installation)\n# Send SIGUSR1 to the Fluentd process\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To confirm that your data has been uploaded successfully, issue the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td tables"]}," command as follows:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td tables\n+------------+------------+------+-----------+\n| Database   | Table      | Type | Count     |\n+------------+------------+------+-----------+\n| test_db    | follow     | log  | 1         |\n+------------+------------+------+-----------+\n"},"children":[]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["The first argument of post() determines the database name and table name. If you specify ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["td.test\\_db.test\\_table"]},", the data is imported into the table *test_table* within the database *test_db*. They are automatically created at upload time."]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"production-deployments","__idx":21},"children":["Production Deployments"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"high-availability-configurations-of-fluentd","__idx":22},"children":["High-Availability Configurations of Fluentd"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For high-traffic websites (more than 5 application nodes), use a high availability configuration of Fluentd to improve data transfer reliability and query performance."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/integration-hub/streaming/td-agent/configuring-td-agent-for-high-availability"},"children":["High-Availability Configurations of Fluentd"]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"monitoring-fluentd","__idx":23},"children":["Monitoring Fluentd"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Monitoring Fluentd itself is also important. Refer to the following document for general monitoring methods for Fluentd."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/integration-hub/streaming/td-agent/monitoring-td-agent"},"children":["Monitoring Fluentd"]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Fluentd is fully open-sourced under the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"http://fluentd.org/"},"children":["Fluentd project"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"next-steps","__idx":24},"children":["Next Steps"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["We offer a schema mechanism that is more flexible than that of traditional RDBMSs. For queries, we leverage the Hive and Trino Query Languages."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/data-workbench/databases/schema-management"},"children":["Schema Management"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/data-workbench/queries/trino/writing_trino_queries"},"children":["Trino Query Language"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/data-workbench/queries/hive/writing_hive_queries"},"children":["Hive Query Language"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/apis/td-api"},"children":["Programmatic Access with REST API and its Bindings"]}]}]}]},"headings":[{"value":"Node.js Apps Import Integration","id":"nodejs-apps-import-integration","depth":1},{"value":"Prerequisites","id":"prerequisites","depth":2},{"value":"Installing Fluentd","id":"installing-fluentd","depth":2},{"value":"Fluentd (fluent-package) Install Options","id":"fluentd-fluent-package-install-options","depth":2},{"value":"RHEL/CentOS/Rocky Linux","id":"rhelcentosrocky-linux","depth":3},{"value":"Ubuntu","id":"ubuntu","depth":3},{"value":"Debian","id":"debian","depth":3},{"value":"Amazon Linux","id":"amazon-linux","depth":3},{"value":"Windows","id":"windows","depth":3},{"value":"macOS","id":"macos","depth":3},{"value":"Starting the Service","id":"starting-the-service","depth":2},{"value":"Linux","id":"linux","depth":3},{"value":"Windows","id":"windows-1","depth":3},{"value":"macOS (gem installation)","id":"macos-gem-installation","depth":3},{"value":"Modifying fluentd.conf","id":"modifying-fluentdconf","depth":2},{"value":"Using fluent-logger-node","id":"using-fluent-logger-node","depth":2},{"value":"Obtaining the Most Recent Version","id":"obtaining-the-most-recent-version","depth":4},{"value":"A Sample Application","id":"a-sample-application","depth":4},{"value":"package.json","id":"packagejson","depth":4},{"value":"web.js","id":"webjs","depth":4},{"value":"Confirming Data Import","id":"confirming-data-import","depth":2},{"value":"Production Deployments","id":"production-deployments","depth":2},{"value":"High-Availability Configurations of Fluentd","id":"high-availability-configurations-of-fluentd","depth":3},{"value":"Monitoring Fluentd","id":"monitoring-fluentd","depth":3},{"value":"Next Steps","id":"next-steps","depth":2}],"frontmatter":{"seo":{"title":"Node.js Apps Import Integration"}},"lastModified":"2026-06-01T09:09:59.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/int/node-js-apps-import-integration","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}