{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":["admonition"]},"redocly_category":"Products","product_name":"Integration Hub","type":"markdown"},"seo":{"title":"Fluentd Using Scribe Protocol to Store Logs","description":"Learn how to use Fluentd with Scribe protocol for log storage, including configuration, input setup, and message handling with JSON strings.","siteUrl":"https://docs.treasure.ai","lang":"en-US","jsonLd":{"@context":"https://schema.org","@graph":[{"@type":"Organization","@id":"https://www.treasure.ai/","name":"Treasure AI","url":"https://www.treasure.ai/","logo":"https://www.treasure.ai/hubfs/assets/images/logos/primary-logo.svg"},{"@type":"WebSite","@id":"https://docs.treasure.ai/#website","name":"Treasure AI Documentation","url":"https://docs.treasure.ai/","inLanguage":["en","ja"],"publisher":{"@id":"https://www.treasure.ai/"}}]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"fluentd-using-scribe-protocol-to-store-logs","__idx":0},"children":["Fluentd Using Scribe Protocol to Store Logs"]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"info"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["td-agent was discontinued in December 2023 and has been replaced by ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["fluent-package"]},". The fluent-package is the official successor maintained by the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://www.cncf.io/projects/"},"children":["Cloud Native Computing Foundation"]},"."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Fluentd understands the Scribe protocol (Thrift-based). Fluentd can co-exist or replace your existing Scribe infrastructure."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["See ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://github.com/fluent/fluent-plugin-scribe"},"children":["fluent-plugin-scribe"]},". The plugin also supports Scribe ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["output"]},"."]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#prerequisites"},"children":["Prerequisites"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#specifying-use-of-scribe-input"},"children":["Specifying Use of Scribe Input"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#messages-as-json-string"},"children":["Messages as JSON String"]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#other-formats"},"children":["Other Formats"]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"prerequisites","__idx":1},"children":["Prerequisites"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Basic knowledge of Treasure Data, including the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/tools/cli-and-sdks/td-toolbelt"},"children":["TD Toolbelt"]},"."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Basic knowledge of Fluentd."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"specifying-use-of-scribe-input","__idx":2},"children":["Specifying Use of Scribe Input"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The following configuration enables the Scribe input plugin:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"conf","header":{"controls":{"copy":{}}},"source":"<source>\n  @type scribe\n  port 1463\n</source>\n","lang":"conf"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["category"]}," field of the Scribe protocol is used as a ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["tag"]}," for Fluentd. If you want to store your data in the www_access table within the test_db database, send your LogEntry as follows (Ruby example)."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"ruby","header":{"controls":{"copy":{}}},"source":"LogEntry.new\nentry.category = 'td.test_db.www_access'\nentry.message = 'abcde'\nclient.Log([entry])\n","lang":"ruby"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["This message is organized as follows."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"ruby","header":{"controls":{"copy":{}}},"source":"{\n  message => 'abcde'\n}\n","lang":"ruby"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"messages-as-json-string","__idx":3},"children":["Messages as JSON String"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The message field of semi-structured data often contains JSON strings. Fluentd uses the ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["msg_format"]}," option to parse the JSON string in-place."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"conf","header":{"controls":{"copy":{}}},"source":"<source>\n  @type scribe\n  port 1463\n  msg_format json\n</source>\n","lang":"conf"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Send records as follows. Note that entry.message is a JSON string."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"ruby","header":{"controls":{"copy":{}}},"source":"entry = LogEntry.new\nentry.category = 'td.test_db.www_access'\nentry.message = {'a' => 'b', 'c' => d}.to_json\nclient.Log([entry])\n","lang":"ruby"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["This message is organized as follows."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"ruby","header":{"controls":{"copy":{}}},"source":"{\n  'a' => 'b',\n  'c' => 'd',\n}\n","lang":"ruby"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"other-formats","__idx":4},"children":["Other Formats"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The ",{"$$mdtype":"Tag","name":"em","attributes":{},"children":["msg_format"]}," option supports ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["text"]},", ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["json"]},", and ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["url_param"]},". The format for ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["url_param"]}," is as follows."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["key1=val1&key2=val2&key3=val3"]}]}]},"headings":[{"value":"Fluentd Using Scribe Protocol to Store Logs","id":"fluentd-using-scribe-protocol-to-store-logs","depth":1},{"value":"Prerequisites","id":"prerequisites","depth":2},{"value":"Specifying Use of Scribe Input","id":"specifying-use-of-scribe-input","depth":2},{"value":"Messages as JSON String","id":"messages-as-json-string","depth":2},{"value":"Other Formats","id":"other-formats","depth":2}],"frontmatter":{"seo":{"title":"Fluentd Using Scribe Protocol to Store Logs","description":"Learn how to use Fluentd with Scribe protocol for log storage, including configuration, input setup, and message handling with JSON strings."}},"lastModified":"2026-06-17T07:22:53.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/products/customer-data-platform/integration-hub/streaming/td-agent/td-agent-using-scribe-protocol-to-store-logs","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}