{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":["admonition","partial"]},"redocly_category":"Integrations","type":"markdown"},"seo":{"title":"Apache Access Logs Import Integration","description":"Treasure Data Product Documentation · Collect and Unify · Segment and Activate · Experiment and Analyze · Decisioning Automate with AI Scale and Trust.","siteUrl":"https://docs.treasuredata.com","lang":"en-US","llmstxt":{"hide":false,"sections":[{"title":"Table of contents","includeFiles":["**/*"],"excludeFiles":[]}],"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"apache-access-logs-import-integration","__idx":0},"children":["Apache Access Logs Import Integration"]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["td-agent was discontinued in December 2023 and has been replaced by ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["fluent-package"]},". The fluent-package is the official successor maintained by the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://www.cncf.io/projects/"},"children":["Cloud Native Computing Foundation"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["You can tail Apache logs using Fluentd to continuously import the access logs into the cloud."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"installing-fluentd","__idx":1},"children":["Installing Fluentd"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"fluentd-fluent-package-install-options","__idx":2},"children":["Fluentd (fluent-package) Install Options"]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["td-agent was discontinued in December 2023 and has been replaced by ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["fluent-package"]},". The fluent-package is the official successor maintained by the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://www.cncf.io/projects/"},"children":["Cloud Native Computing Foundation"]},". For migration guidance from td-agent, see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.fluentd.org/installation/install-fluent-package"},"children":["Fluentd Installation Guide"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To install ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["fluent-package"]},", run one of the following commands based on your environment."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"rhelcentosrocky-linux","__idx":3},"children":["RHEL/CentOS/Rocky Linux"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# fluent-package 6 LTS (recommended)\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-redhat-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"ubuntu","__idx":4},"children":["Ubuntu"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Ubuntu 24.04 Noble - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-ubuntu-noble-fluent-package6-lts.sh | sh\n\n# Ubuntu 22.04 Jammy - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-ubuntu-jammy-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"debian","__idx":5},"children":["Debian"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Debian Bookworm - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-debian-bookworm-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"amazon-linux","__idx":6},"children":["Amazon Linux"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# Amazon Linux 2023 - fluent-package 6 LTS\ncurl -fsSL https://fluentd.cdn.cncf.io/sh/install-amazon2023-fluent-package6-lts.sh | sh\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"windows","__idx":7},"children":["Windows"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Download the MSI installer from:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://fluentd.cdn.cncf.io/lts/6/windows/index.html"},"children":["fluent-package 6 LTS for Windows"]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After installation:"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Edit the configuration file at ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["C:/opt/fluent/etc/fluent/fluentd.conf"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Start the service using ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["net start fluentdwinsvc"]}," or via Services administrative tool"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"macos","__idx":8},"children":["macOS"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["fluent-package for macOS is planned to be available via Homebrew. For current installation options, see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.fluentd.org/installation"},"children":["Fluentd Installation Guide"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"starting-the-service","__idx":9},"children":["Starting the Service"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After installation, start and verify the Fluentd service."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"linux","__idx":10},"children":["Linux"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"sudo systemctl start fluentd.service\nsudo systemctl status fluentd.service\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The configuration file is located at ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/etc/fluent/fluentd.conf"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"windows-1","__idx":11},"children":["Windows"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"cmd","header":{"controls":{"copy":{}}},"source":"net start fluentdwinsvc\n","lang":"cmd"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The configuration file is located at ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["C:\\opt\\fluent\\etc\\fluent\\fluentd.conf"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"macos-gem-installation","__idx":12},"children":["macOS (gem installation)"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"fluentd -c /path/to/fluentd.conf\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["For more details, see the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.fluentd.org/"},"children":["Fluentd Documentation"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"modifying-fluentdconf","__idx":13},"children":["Modifying fluentd.conf"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Specify your authentication key by setting the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["apikey"]}," option. You can view your API key from the Treasure Console."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To modify the file, go to ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/etc/fluent/fluentd.conf"]}," (for fluent-package). Set the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["apikey"]}," option in your fluentd.conf file."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Where ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["YOUR_API_KEY"]}," is your actual apikey string."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"conf","header":{"controls":{"copy":{}}},"source":"# Tailing the Apache Log\n<source>\n  @type tail\n  path /var/log/httpd-access.log\n  pos_file /var/log/fluent/httpd-access.pos\n  tag td.production.access\n  <parse>\n    @type apache2\n  </parse>\n</source>\n\n# Treasure Data Input and Output\n<match td.*.*>\n  @type tdlog\n  endpoint api.treasuredata.com\n  apikey YOUR_API_KEY\n  auto_create_table\n  use_ssl true\n  <buffer>\n    @type file\n    path /var/log/fluent/buffer/td\n  </buffer>\n</match>\n","lang":"conf"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Restart the Fluentd service when the following lines are in place."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"sudo systemctl restart fluentd.service\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Fluentd keeps tailing the log, buffers the log (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/var/log/fluent/buffer/td"]},"), and automatically uploads the log into the cloud."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"confirming-data-import","__idx":14},"children":["Confirming Data Import"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Sending a SIGUSR1 signal flushes Fluentd's buffer and the upload starts immediately."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"bash","header":{"controls":{"copy":{}}},"source":"# generate access logs\n$ curl http://host:port/\n\n# flush the buffer\n$ kill -USR1 $(cat /var/run/fluent/fluentd.pid)\n","lang":"bash"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["To confirm that your data has been uploaded successfully, issue the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["td tables"]}," command as follows."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"$ td tables\n+------------+------------+------+-----------+\n| Database   | Table      | Type | Count     |\n+------------+------------+------+-----------+\n| production | access     | log  | 1         |\n+------------+------------+------+-----------+\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Check ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/var/log/fluent/fluentd.log"]}," if it's not working correctly. The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["fluentd:fluentd"]}," user must have permission to read the logs."]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Fluentd handles log-rotation"]},". Fluentd keeps a record of the last position of the log, ensuring that each line is read exactly one time even if the Fluentd process goes down. However, because the information is kept in a file, the \"exactly once\" guarantee breaks down if the file becomes corrupted."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"next-steps","__idx":15},"children":["Next Steps"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["We offer a schema mechanism that is more flexible than that of traditional RDBMSs. For queries, we leverage the Hive Query Language."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/data-workbench/databases/schema-management"},"children":["Schema Management"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/products/customer-data-platform/data-workbench/queries/hive/writing_hive_queries"},"children":["Writing Hive Queries"]}]}]}]},"headings":[{"value":"Apache Access Logs Import Integration","id":"apache-access-logs-import-integration","depth":1},{"value":"Installing Fluentd","id":"installing-fluentd","depth":2},{"value":"Fluentd (fluent-package) Install Options","id":"fluentd-fluent-package-install-options","depth":2},{"value":"RHEL/CentOS/Rocky Linux","id":"rhelcentosrocky-linux","depth":3},{"value":"Ubuntu","id":"ubuntu","depth":3},{"value":"Debian","id":"debian","depth":3},{"value":"Amazon Linux","id":"amazon-linux","depth":3},{"value":"Windows","id":"windows","depth":3},{"value":"macOS","id":"macos","depth":3},{"value":"Starting the Service","id":"starting-the-service","depth":2},{"value":"Linux","id":"linux","depth":3},{"value":"Windows","id":"windows-1","depth":3},{"value":"macOS (gem installation)","id":"macos-gem-installation","depth":3},{"value":"Modifying fluentd.conf","id":"modifying-fluentdconf","depth":2},{"value":"Confirming Data Import","id":"confirming-data-import","depth":2},{"value":"Next Steps","id":"next-steps","depth":2}],"frontmatter":{"seo":{"title":"Apache Access Logs Import Integration"}},"lastModified":"2026-06-01T09:09:59.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/int/apache-access-logs-import-integration","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}