{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":["admonition","partial"]},"redocly_category":"Integrations","type":"markdown"},"seo":{"title":"Perl Apps Import Integration","description":"Treasure Data Product Documentation · Collect and Unify · Segment and Activate · Experiment and Analyze · Decisioning Automate with AI Scale and Trust.","siteUrl":"https://docs.treasuredata.com","lang":"en-US","llmstxt":{"hide":false,"sections":[{"title":"Table of contents","includeFiles":["**/*"],"excludeFiles":[]}],"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"perl-apps-import-integration","__idx":0},"children":["Perl Apps Import Integration"]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["td-agent was discontinued in December 2023 and has been replaced by ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["fluent-package"]},". The fluent-package is the official successor maintained by the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://www.cncf.io/projects/"},"children":["Cloud Native Computing Foundation"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Treasure Data provides ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/integration-hub/streaming/td-agent/about-treasure-data-s-server-side-agent"},"children":["Fluentd"]}," to collect server-side logs and events and to seamlessly import the data from Perl applications."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"prerequisites","__idx":1},"children":["Prerequisites"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Basic knowledge of Perl and CPAN."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Basic knowledge of Treasure Data."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Perl 5.26 or higher (for local testing)."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"installing-fluentd","__idx":2},"children":["Installing Fluentd"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Install Fluentd (fluent-package) on your application servers. Fluentd sits within your application servers, focusing on uploading application logs to the cloud."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/image2020-12-2_14-12-51.1754f4e3154420d54cc976b16d454ef773c83ad5fc4b8580d99a87302c614932.23757061.png","alt":""},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"http://github.com/fluent/fluent-logger-perl"},"children":["fluent-logger-perl"]}," library enables Perl applications to post records to their local Fluentd. Fluentd, in turn, uploads the data to the cloud every 5 minutes. Because the daemon runs on a local node, the logging latency is negligible."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"fluentd-fluent-package-install-options","__idx":3},"children":["Fluentd (fluent-package) Install Options"]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["td-agent was discontinued in December 2023 and has been replaced by ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["fluent-package"]},". The fluent-package is the official successor maintained by the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://www.cncf.io/projects/"},"children":["Cloud Native Computing Foundation"]},". For migration guidance from td-agent, see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.fluentd.org/installation/install-fluent-package"},"children":["Fluentd Installation Guide"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To install ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["fluent-package"]},", run one of the following commands based on your environment."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"rhelcentosrocky-linux","__idx":4},"children":["RHEL/CentOS/Rocky Linux"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# fluent-package 6 LTS (recommended)\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-redhat-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"ubuntu","__idx":5},"children":["Ubuntu"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Ubuntu 24.04 Noble - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-ubuntu-noble-fluent-package6-lts.sh | sh\n\n# Ubuntu 22.04 Jammy - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-ubuntu-jammy-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"debian","__idx":6},"children":["Debian"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Debian Bookworm - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-debian-bookworm-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"amazon-linux","__idx":7},"children":["Amazon Linux"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Amazon Linux 2023 - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-amazon2023-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"windows","__idx":8},"children":["Windows"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Download the MSI installer from:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://fluentd.cdn.cncf.io/lts/6/windows/index.html"},"children":["fluent-package 6 LTS for Windows"]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After installation:"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Edit the configuration file at ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["C:/opt/fluent/etc/fluent/fluentd.conf"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Start the service using ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["net start fluentdwinsvc"]}," or via Services administrative tool"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"macos","__idx":9},"children":["macOS"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["fluent-package for macOS is planned to be available via Homebrew. For current installation options, see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.fluentd.org/installation"},"children":["Fluentd Installation Guide"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"starting-the-service","__idx":10},"children":["Starting the Service"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After installation, start and verify the Fluentd service."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"linux","__idx":11},"children":["Linux"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"sudo systemctl start fluentd.service\nsudo systemctl status fluentd.service\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The configuration file is located at ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/etc/fluent/fluentd.conf"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"windows-1","__idx":12},"children":["Windows"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"cmd","header":{"controls":{"copy":{}}},"source":"net start fluentdwinsvc\n","lang":"cmd"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The configuration file is located at ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["C:\\opt\\fluent\\etc\\fluent\\fluentd.conf"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"macos-gem-installation","__idx":13},"children":["macOS (gem installation)"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"fluentd -c /path/to/fluentd.conf\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For more details, see the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.fluentd.org/"},"children":["Fluentd Documentation"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"modifying-fluentdconf","__idx":14},"children":["Modifying fluentd.conf"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Specify your API key by setting the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["apikey"]}," option in your ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/etc/fluent/fluentd.conf"]}," file (for fluent-package). You can view your API key from your profile in Treasure Console."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"conf","header":{"controls":{"copy":{}}},"source":"# Treasure Data Input and Output\n<source>\n  @type forward\n  port 24224\n</source>\n\n<match td.*.*>\n  @type tdlog\n  endpoint api.treasuredata.com\n  apikey YOUR_API_KEY\n  auto_create_table\n  use_ssl true\n  <buffer>\n    @type file\n    path /var/log/fluent/buffer/td\n  </buffer>\n</match>\n","lang":"conf"},"children":[]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["YOUR_API_KEY"]}," should be your actual apikey string. You can retrieve your API key from your profiles in Treasure Console. Using a ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/my-settings/getting-your-api-keys"},"children":["write-only API key"]}," is recommended."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Restart the Fluentd service when the following lines are in place."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Linux\nsudo systemctl restart fluentd.service\n\n# macOS (gem installation)\n# Restart the Fluentd process manually\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Fluentd now accepts data via port 24224, buffers the data (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/var/log/fluent/buffer/td"]},"), and automatically uploads the data into the cloud."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"using-fluent-logger-perl","__idx":15},"children":["Using fluent-logger-perl"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["First, install the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["Fluent::Logger"]}," library via CPAN."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ cpan\ncpan[1]> install Fluent::Logger\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Next, initialize and post the records as follows."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"testpl","__idx":16},"children":["test.pl"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"perl","header":{"controls":{"copy":{}}},"source":"use Fluent::Logger;\nmy $logger = Fluent::Logger->new(\n    host => '127.0.0.1',\n    port => 24224,\n    tag_prefix => 'td.test_db',\n);\n$logger->post(\"test_table\", { \"entry1\" => \"value1\", \"entry2\" => 2 });\n","lang":"perl"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"confirming-data-import","__idx":17},"children":["Confirming Data Import"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["First, execute the preceding program."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"$ perl test.pl\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Sending a SIGUSR1 signal flushes Fluentd's buffer. Upload starts immediately."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Linux\n$ kill -USR1 $(cat /var/run/fluent/fluentd.pid)\n\n# macOS (gem installation)\n# Send SIGUSR1 to the Fluentd process\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To confirm that your data has been uploaded successfully, issue the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["td tables"]}," command as follows:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td tables\n+------------+------------+------+-----------+\n| Database   | Table      | Type | Count     |\n+------------+------------+------+-----------+\n| test_db    | test_table | log  | 1         |\n+------------+------------+------+-----------+\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The first argument of post() determines the database name and table name. If you specify ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["td.test\\_db.test\\_table"]},", the data is imported into the table *test_table* within the database *test_db*. They are automatically created at upload time."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"production-deployments","__idx":18},"children":["Production Deployments"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"high-availability-configurations-of-fluentd","__idx":19},"children":["High-Availability Configurations of Fluentd"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For high-traffic websites (more than 5 application nodes), use a high availability configuration of Fluentd to improve data transfer reliability and query performance."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/integration-hub/streaming/td-agent/configuring-td-agent-for-high-availability"},"children":["High-Availability Configurations of Fluentd"]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"monitoring-fluentd","__idx":20},"children":["Monitoring Fluentd"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Monitoring Fluentd itself is also important. For general monitoring methods for Fluentd see the following:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/integration-hub/streaming/td-agent/monitoring-td-agent"},"children":["Monitoring Fluentd"]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Fluentd is fully open-sourced under the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"http://fluentd.org/"},"children":["Fluentd project"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"next-steps","__idx":21},"children":["Next Steps"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["We offer a schema mechanism that is more flexible than that of traditional RDBMSs. For queries, we leverage the Hive and Trino Query Languages."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/data-workbench/databases/schema-management"},"children":["Schema Management"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/data-workbench/queries/trino/writing_trino_queries"},"children":["Trino Query Language"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/data-workbench/queries/hive/writing_hive_queries"},"children":["Hive Query Language"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/apis/td-api"},"children":["Programmatic Access with REST API and its Bindings"]}]}]}]},"headings":[{"value":"Perl Apps Import Integration","id":"perl-apps-import-integration","depth":1},{"value":"Prerequisites","id":"prerequisites","depth":2},{"value":"Installing Fluentd","id":"installing-fluentd","depth":2},{"value":"Fluentd (fluent-package) Install Options","id":"fluentd-fluent-package-install-options","depth":2},{"value":"RHEL/CentOS/Rocky Linux","id":"rhelcentosrocky-linux","depth":3},{"value":"Ubuntu","id":"ubuntu","depth":3},{"value":"Debian","id":"debian","depth":3},{"value":"Amazon Linux","id":"amazon-linux","depth":3},{"value":"Windows","id":"windows","depth":3},{"value":"macOS","id":"macos","depth":3},{"value":"Starting the Service","id":"starting-the-service","depth":2},{"value":"Linux","id":"linux","depth":3},{"value":"Windows","id":"windows-1","depth":3},{"value":"macOS (gem installation)","id":"macos-gem-installation","depth":3},{"value":"Modifying fluentd.conf","id":"modifying-fluentdconf","depth":2},{"value":"Using fluent-logger-perl","id":"using-fluent-logger-perl","depth":2},{"value":"test.pl","id":"testpl","depth":3},{"value":"Confirming Data Import","id":"confirming-data-import","depth":2},{"value":"Production Deployments","id":"production-deployments","depth":2},{"value":"High-Availability Configurations of Fluentd","id":"high-availability-configurations-of-fluentd","depth":3},{"value":"Monitoring Fluentd","id":"monitoring-fluentd","depth":3},{"value":"Next Steps","id":"next-steps","depth":2}],"frontmatter":{"seo":{"title":"Perl Apps Import Integration"}},"lastModified":"2026-06-01T09:09:59.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/int/perl-apps-import-integration","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}