{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-@l10n/ja/sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":["admonition"]},"type":"markdown"},"seo":{"title":"What is incremental_update?","description":"Treasure Data Product Documentation · Collect and Unify · Segment and Activate · Experiment and Analyze · Decisioning Automate with AI Scale and Trust.","siteUrl":"https://docs.treasuredata.com","lang":"en-US","llmstxt":{"hide":false,"sections":[{"title":"Table of contents","includeFiles":["**/*"],"excludeFiles":[]}],"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"what-is-incremental_update","__idx":0},"children":["What is incremental_update?"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["incremental_update"]}," is a mechanism that reduces processing time by leveraging the results of the previous ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Unification"]}," process and applying stitching only to updated records."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"how-does-incremental_update-improve-efficiency","__idx":1},"children":["How does incremental_update improve efficiency?"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["When ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_columns: [time]"]}," is specified, the Unification Algorithm combines the previous final graph with the graph generated from newly added records. This allows the process to start from a state where most of the stitching is already complete, leading to faster convergence."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["When ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_columns: [time]"]}," is specified, the enrichment process applies the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["canonical_id"]}," only to the delta records. This significantly shortens processing time compared to enriching the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["canonical_id"]}," for all records."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"dataset","__idx":2},"children":["Dataset"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Using the table from ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/ja/products/customer-data-platform/id-unification/p3_canonical_id_learning"},"children":["Example"]}," as a base, we will insert records during the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["incremental_update"]}," process and observe how the update is applied."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"table-used-in-example1","__idx":3},"children":["Table Used in Example1"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Assume the data from Example1 is stored in the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["test_id_unification_ex5"]}," database."]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"date"},"children":["date"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_aaa"},"children":["site_aaa"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_aaa"},"children":["site_aaa"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_xxx"},"children":["site_xxx"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_xxx"},"children":["site_xxx"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_yyy"},"children":["site_yyy"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_yyy"},"children":["site_yyy"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_zzz"},"children":["site_zzz"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_zzz"},"children":["site_zzz"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["month"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["day"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_client_id"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_global_id"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_client_id"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_global_id"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_client_id"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_global_id"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_client_id"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_global_id"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["1"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["5"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["aaa_001"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_001"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["yyy_001"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_001"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["15"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["aaa_001"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_002"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["zzz_001"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_002"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["25"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["aaa_001"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_003"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["2"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["5"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["aaa_001"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_004"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["xxx_001"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_004"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["15"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["xxx_001"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_005"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["yyy_002"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_005"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["25"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["yyy_002"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_006"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["zzz_003"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_006"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["5"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["zzz_003"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_007"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["15"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["xxx_002"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_008"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["zzz_003"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_008"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["25"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["aaa_002"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_009"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["xxx_002"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_009"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["4"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["5"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["aaa_002"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_010"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["yyy_003"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_010"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["15"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["yyy_003"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_011"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["zzz_004"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_011"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["25"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["xxx_003"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_012"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["zzz_004"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_012"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["5"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["5"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["aaa_003"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_013"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["xxx_003"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_013"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["15"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["aaa_003"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_014"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["25"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["aaa_003"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_015"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["yyy_004"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_015"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["zzz_005"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_015"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["6"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["5"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["aaa_003"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_016"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["xxx_004"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_016"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["15"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["xxx_004"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_017"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["zzz_005"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_017"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["25"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["yyy_005"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_018"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["zzz_005"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_018"]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"records-to-be-added-later","__idx":4},"children":["Records to be Added Later"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"date"},"children":["date"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_aaa"},"children":["site_aaa"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_aaa"},"children":["site_aaa"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_xxx"},"children":["site_xxx"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_xxx"},"children":["site_xxx"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_yyy"},"children":["site_yyy"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_yyy"},"children":["site_yyy"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":""},"children":[]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_zzz"},"children":["site_zzz"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"site_zzz"},"children":["site_zzz"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["month"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["day"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_client_id"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_global_id"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_client_id"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_global_id"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_client_id"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_global_id"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_client_id"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["td_global_id"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["7"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["5"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["aaa_004"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_017"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["xxx_005"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_018"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["yyy_006"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_019"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["zzz_006"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_018"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["15"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["aaa_004"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_018"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["zzz_007"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_018"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["25"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["8"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["5"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["aaa_005"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_018"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["xxx_005"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_018"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["15"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["xxx_006"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_019"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["yyy_006"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_019"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["zzz_008"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_019"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["25"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["aaa_005"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_019"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["yyy_007"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["3rd_019"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[]}]}]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["We will add the above records midway and observe the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["incremental_update"]}," process."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"workflow-example","__idx":5},"children":["Workflow example"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"id_unification_ex5dig","__idx":6},"children":["id_unification_ex5.dig"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"timezone: UTC # Asia/Tokyo\nschedule:\n  daily>: 09:00:00\n\n+call_unification:\n  http_call>: https://api-cdp.treasuredata.com/unifications/workflow_call\n  headers:\n    - authorization: ${secret:td.apikey}\n  method: POST\n  retry: true\n  content_format: json\n  content:\n\n    run_canonical_ids: true\n    run_enrichments: true\n    run_master_tables: true\n\n    full_refresh: false\n    keep_debug_tables: true\n\n    unification:\n      !include : unification_ex5.yml\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"unification_ex5yml","__idx":7},"children":["unification_ex5.yml"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"name: test_id_unification_ex5\n\nkeys:\n  - name: td_client_id\n  - name: td_global_id\n\ntables:\n  - database: test_id_unification_ex5\n    table: site_aaa\n    incremental_columns: [time]\n    key_columns:\n      - {column: td_client_id, key: td_client_id}\n      - {column: td_global_id, key: td_global_id}\n  - database: test_id_unification_ex5\n    table: site_xxx\n    incremental_columns: [time]\n    key_columns:\n      - {column: td_client_id, key: td_client_id}\n      - {column: td_global_id, key: td_global_id}\n\n  - database: test_id_unification_ex5\n    table: site_yyy\n    incremental_columns: [time]\n    key_columns:\n      - {column: td_client_id, key: td_client_id}\n      - {column: td_global_id, key: td_global_id}\n\n  - database: test_id_unification_ex5\n    table: site_zzz\n    incremental_columns: [time]\n    key_columns:\n      - {column: td_client_id, key: td_client_id}\n      - {column: td_global_id, key: td_global_id}\n\ncanonical_ids:\n  - name: unified_cookie_id\n    merge_by_keys: [td_client_id, td_global_id]\n    merge_iterations: 5\n    incremental_merge_iterations: 3\n\nmaster_tables:\n  - name: master_table_ex5\n    canonical_id: unified_cookie_id\n    attributes:\n    - name: td_client_id\n      array_elements: 5\n      source_columns:\n        - {table: site_aaa, order: last, order_by: time, priority: 1}\n        - {table: site_xxx, order: last, order_by: time, priority: 1}\n        - {table: site_yyy, order: last, order_by: time, priority: 1}\n        - {table: site_zzz, order: last, order_by: time, priority: 1}\n    - name: td_global_id\n      array_elements: 5\n      source_columns:\n        - {table: site_aaa, order: last, order_by: time, priority: 1}\n        - {table: site_xxx, order: last, order_by: time, priority: 1}\n        - {table: site_yyy, order: last, order_by: time, priority: 1}\n        - {table: site_zzz, order: last, order_by: time, priority: 1}\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"settings-for-incremental_update","__idx":8},"children":["Settings for incremental_update"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"full_refresh-false","__idx":9},"children":["full_refresh: false"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"  content:\n\n    run_canonical_ids: true\n    run_enrichments: true\n    run_master_tables: true\n\n    full_refresh: false\n    keep_debug_tables: true\n\n    unification:\n      !include : unification_ex5.yml\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["By setting ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["full_refresh: false"]}," in the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["content:"]}," section of the dig file, you enable the configuration for ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_update"]},". However, note that ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_update"]}," is not performed on a daily basis. Instead, a ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["full_refresh"]}," (processing all records as usual) is performed once every three days."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"incremental_columns","__idx":10},"children":["incremental_columns:"]},{"$$mdtype":"Tag","name":"Admonition","attributes":{"type":"warning","name":"Warning"},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Currently, any setting other than ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_columns: [time]"]}," will result in all processes being executed as ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["full_refresh"]},". Avoid using other configurations."]}]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"tables:\n  - database: test_id_unification_ex5\n    table: site_aaa\n    incremental_columns: [time]\n    key_columns:\n      - {column: td_client_id, key: td_client_id}\n      - {column: td_global_id, key: td_global_id}\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["In the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["tables:"]}," section of the YAML file, you can configure ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_columns: [column1, column2,...]"]}," for each table. When the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["+get_next_high_water_mark"]}," task records where processing left off, the order specified in this option will be used for sorting:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"SQL","header":{"controls":{"copy":{}}},"source":"ORDER BY column1 DESC, column2 DESC,...\n","lang":"SQL"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The first row in this sorted order (the most recent record) will have its column1, column2,... values recorded."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If this option is not set, the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["+get_next_high_water_mark"]}," task will not run. The Unification Algorithm will operate in a manner similar to ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["full_refresh"]},", and all records in the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["enriched_"]}," tables will be replaced."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"incremental_columns-time","__idx":11},"children":["incremental_columns: [time]"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["When ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_columns: [time]"]}," is specified (only the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["time"]}," column is defined), the process qualifies as ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_update"]},". The Unification Algorithm considers only newly added records, and instead of replacing entire ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["enriched_"]}," tables, enrichment is performed on newly added records and appended to the table. As of now, only the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["[time]"]}," setting enables efficient incremental updates, so keep this in mind."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"incremental_merge_iterations","__idx":12},"children":["incremental_merge_iterations:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"canonical_ids:\n  - name: unified_cookie_id\n    merge_by_keys: [td_client_id, td_global_id]\n    merge_iterations: 5\n    incremental_merge_iterations: 3\n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["In the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["canonical_ids:"]}," configuration, the number of iterations for loops during ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_update"]}," can be specified using ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_merge_iterations:"]},". If not set, the default value is ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["2"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"how-to-verify-if-incremental_update-was-performed","__idx":13},"children":["How to Verify if incremental_update Was Performed"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["A ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["full_refresh"]}," is performed once every three days, while ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_update"]}," is executed on other days. On the second or third day of the schedule execution, check the most recent session (history) of the workflow (WF) using the methods below to identify sessions where ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_update"]}," was processed."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"method-1-check-the-extract_and_merge-task","__idx":14},"children":["Method 1: Check the +extract_and_merge Task"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"SQL","header":{"controls":{"copy":{}}},"source":"drop table if exists \"unified_cookie_id_graph_unify_loop_0\";\ncreate table \"unified_cookie_id_graph_unify_loop_0\" with (bucketed_on = array['follower_id'], bucket_count = 512) as\n-- incremental extraction and merge to the previous graph\nselect\n  coalesce(prev.follower_id, next.follower_id) as follower_id,\n  coalesce(prev.follower_ns, next.follower_ns) as follower_ns,\n","lang":"SQL"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["If the comment just above the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["select"]}," statement reads:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"SQL","header":{"controls":{"copy":{}}},"source":"-- incremental extraction and merge to the previous graph\n","lang":"SQL"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["then this WF session performed an ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_update"]},". Conversely, for ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["full_refresh"]},", the comment will read:"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"SQL","header":{"controls":{"copy":{}}},"source":"-- full extraction\n","lang":"SQL"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"method-2-check-if-the-source_key_stats-task-was-executed","__idx":15},"children":["Method 2: Check if the +source_key_stats Task Was Executed"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/8-1-1.c0a099edd6327582c58fa5ccb880ffdbcbf6fd79c20c84ab967c92643ece2699.b7a43e2a.avif","alt":"canonical_id_1"},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["In the TIMELINE or TASKS section of the WF session, if the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["+source_key_stats:"]}," task was executed, it indicates a ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["full_refresh"]},". If this task was not executed, it signifies an ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_update"]},"."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["On days when ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_update"]}," is executed, you can add new records using the ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://github.com/treasure-data/treasure-boxes/blob/master/tool-box/id-unification-samples/queries/insert_samples_ex5.sql"},"children":["SQL for Appending Records"]}," and rerun the WF to confirm the processing of newly added records during the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_update"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"explanation-of-the-unification-algorithm","__idx":16},"children":["Explanation of the Unification Algorithm"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"graph_unify_loop_0","__idx":17},"children":["graph_unify_loop_0"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["In an ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_update"]},", the creation of the initial graph differs. The initial graph is formed by combining the following two graphs:"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["The graph for newly added records"]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["The final graph from the previous execution"]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"1-the-graph-for-newly-added-records","__idx":18},"children":["1. The Graph for Newly Added Records"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/graph_unify_loop_0.46e50526aa92b099b1896d60907469d84f80226d10aa858b8026611e611ed92d.b7a43e2a.avif","alt":"canonical_id_1"},"children":[]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"2-the-final-graph-from-the-previous-execution","__idx":19},"children":["2. The Final Graph from the Previous Execution"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/graph_unify_loop_4.eeb038dad40ab18297837ed8929da6c3bf2680ba3067a9b2626b7d81b27d2f5c.55be243a.avif","alt":"canonical_id_1"},"children":[]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"graph_unify_loop_0-1","__idx":20},"children":["graph_unify_loop_0"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The initial graph for ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_update"]}," is the result of combining these two graphs."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/graph_unify_loop_0.ef5ff3880e9ed2bebd162c3a783164138edc6bbfbeb7535283ce74fd5a9abb7d.1622d91e.avif","alt":"canonical_id_1"},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Starting from this graph allows the algorithm to converge with fewer iterations. In most cases, during the first loop, when the leader is replaced, it converges to ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["aaa_001"]}," (see Example 2), bringing it closer to convergence."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"graph_unify_loop_1-and-similar-graphs-for-2-etc","__idx":21},"children":["graph_unify_loop_1 (and Similar Graphs for 2, etc.)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"img","attributes":{"src":"/assets/graph_unify_loop_1.b592f2309152450e6335ca66d049c0b3b77649338f04db1a7c6488fc527c9e63.1622d91e.avif","alt":"canonical_id_1"},"children":[]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["In practice, it is confirmed that the graph converges in the first iteration."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"output","__idx":22},"children":["Output"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"master_table","__idx":23},"children":["master_table"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Since the newly added records also belong to the same individual, the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["master_table"]}," consists of a single record. The ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["canonical_id"]}," is generated based on ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["aaa_001"]},", so its value remains unchanged from the previous execution."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":4,"id":"result-example","__idx":24},"children":["Result Example"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"unified_cookie_id"},"children":["unified_cookie_id"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"td_client_id"},"children":["td_client_id"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"td_global_id"},"children":["td_global_id"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Su-bHvUu9NN_"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["[\"yyy_007\", \"aaa_005\", \"xxx_006\", \"yyy_006\", \"zzz_008\"]"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["[\"3rd_019\", \"3rd_019\", \"3rd_019\", \"3rd_019\", \"3rd_019\"]"]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"enriched_-table","__idx":25},"children":["enriched_ Table"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Under the configuration ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_columns: [time]"]},", when an ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_update"]}," is performed, ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["canonical_id"]}," is not assigned to all records in the ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["enriched_"]}," Table with a Replace operation. Instead, ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["canonical_id"]}," is assigned only to the updated records, and these are appended to the existing ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["enriched_"]}," Table. This drastically reduces the time required for the enrichment task."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"limitation-for-incremental_update","__idx":26},"children":["Limitation for incremental_update"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"when-records-from-the-past-are-deleted-or-updated","__idx":27},"children":["When Records from the Past are Deleted or Updated"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["In ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["incremental_update"]},", any deletions or updates to past records are not taken into the ID unification immediately. This requires careful attention. However, since a ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["full_refresh"]}," is performed once every three days, those changes will be addressed during that process."]}]},"headings":[{"value":"What is incremental_update?","id":"what-is-incremental_update","depth":1},{"value":"How does incremental_update improve efficiency?","id":"how-does-incremental_update-improve-efficiency","depth":2},{"value":"Dataset","id":"dataset","depth":2},{"value":"Table Used in Example1","id":"table-used-in-example1","depth":2},{"value":"Records to be Added Later","id":"records-to-be-added-later","depth":2},{"value":"Workflow example","id":"workflow-example","depth":2},{"value":"id_unification_ex5.dig","id":"id_unification_ex5dig","depth":3},{"value":"unification_ex5.yml","id":"unification_ex5yml","depth":3},{"value":"Settings for incremental_update","id":"settings-for-incremental_update","depth":2},{"value":"full_refresh: false","id":"full_refresh-false","depth":3},{"value":"incremental_columns:","id":"incremental_columns","depth":3},{"value":"incremental_columns: [time]","id":"incremental_columns-time","depth":4},{"value":"incremental_merge_iterations:","id":"incremental_merge_iterations","depth":3},{"value":"How to Verify if incremental_update Was Performed","id":"how-to-verify-if-incremental_update-was-performed","depth":2},{"value":"Method 1: Check the +extract_and_merge Task","id":"method-1-check-the-extract_and_merge-task","depth":3},{"value":"Method 2: Check if the +source_key_stats Task Was Executed","id":"method-2-check-if-the-source_key_stats-task-was-executed","depth":3},{"value":"Explanation of the Unification Algorithm","id":"explanation-of-the-unification-algorithm","depth":2},{"value":"graph_unify_loop_0","id":"graph_unify_loop_0","depth":3},{"value":"1. The Graph for Newly Added Records","id":"1-the-graph-for-newly-added-records","depth":4},{"value":"2. The Final Graph from the Previous Execution","id":"2-the-final-graph-from-the-previous-execution","depth":4},{"value":"graph_unify_loop_0","id":"graph_unify_loop_0-1","depth":4},{"value":"graph_unify_loop_1 (and Similar Graphs for 2, etc.)","id":"graph_unify_loop_1-and-similar-graphs-for-2-etc","depth":3},{"value":"Output","id":"output","depth":2},{"value":"master_table","id":"master_table","depth":3},{"value":"Result Example","id":"result-example","depth":4},{"value":"enriched_ Table","id":"enriched_-table","depth":3},{"value":"Limitation for incremental_update","id":"limitation-for-incremental_update","depth":2},{"value":"When Records from the Past are Deleted or Updated","id":"when-records-from-the-past-are-deleted-or-updated","depth":3}],"frontmatter":{"seo":{"title":"What is incremental_update?"}},"lastModified":"2026-02-17T01:46:37.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/ja/products/customer-data-platform/id-unification/p7_incremental","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}