{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":[]},"redocly_category":"Products","product_name":"Integration Hub","type":"markdown"},"seo":{"title":"Using the Delta Sharing Capability in a Custom Script Environment","description":"Treasure Data Product Documentation · Collect and Unify · Segment and Activate · Experiment and Analyze · Decisioning Automate with AI Scale and Trust.","siteUrl":"https://docs.treasuredata.com","lang":"en-US","llmstxt":{"hide":false,"sections":[{"title":"Table of contents","includeFiles":["**/*"],"excludeFiles":[]}],"excludeFiles":[]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"using-the-delta-sharing-capability-in-a-custom-script-environment","__idx":0},"children":["Using the Delta Sharing Capability in a Custom Script Environment"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Here is an example of how to configure Delta Sharing capability with custom Python scripts:"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Create shared data on Databricks."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["After the shared data is configured, a Delta Sharing credential file can be downloaded.",{"$$mdtype":"Tag","name":"br","attributes":{},"children":[]},"See ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://docs.databricks.com/en/delta-sharing/share-data-open.html#delta-sharing-open-sharing-workflow"},"children":["Delta Sharing open sharing workflow"]}," for additional details."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Create a workflow based on the processing needs."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Execute a custom script (python code) from the workflow."]}]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Register Delta Sharing credentials to workflow secrets."]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Use the JSON content of your Delta Sharing credentials to supply a Secret Key."]},{"$$mdtype":"Tag","name":"ol","attributes":{"start":6},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Define a workflow task to call a custom Python script."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["In the following example, note that"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["The Py> operator is used to call a specific method in your Python program."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["The Delta Sharing credentials are passed from the workflow secret to the container using the ${secret:configShare} environment variable."]}]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"yaml","header":{"controls":{"copy":{}}},"source":"timezone: UTC\n+call_exmple:\n  docker:\n    image: 'digdag/digdag-python:3.10'\n  py>: delta_sharing_task.fetch_data\n  _env:\n    configShare: '${secret:configShare}'  \n","lang":"yaml"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["In the Python program, PySpark is used to access shared data on Databricks, and the data can be manipulated as a standard spark dataframe."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"python","header":{"controls":{"copy":{}}},"source":"import sys  \nimport os  \nimport json  \nfrom pyspark.sql import SparkSession  \n  \n  \ndef fetch_data():  \n  # Install and import delta-sharing package  \n  os.system(f'{sys.executable} -m pip install delta-sharing')  \n  import delta_sharing  \n    \n    \n  # Prepare config.share file  \n  with open(\"config.share\", \"w\") as outfile:  \n    outfile.write(os.environ.get('configShare'))  \n      \n      \n  # Fetch shared data on Databricks via Delta-Sharing server by PySpark  \n  table_url = f\"./config.share#your_share.your_database.your_table\"  \n  spark = SparkSession.builder \\  \n    .appName(\"delta-sharing-app\") \\  \n    .master(\"local[*]\") \\  \n    .config('spark.jars.packages', 'org.apache.hadoop:hadoop-azure:3.3.1,io.delta:delta-core_2.12:2.2.0,io.delta:delta-sharing-spark_2.12:0.6.2') \\  \n    .config('spark.sql.extensions', 'io.delta.sql.DeltaSparkSessionExtension') \\  \n    .config('spark.sql.catalog.spark_catalog', 'org.apache.spark.sql.delta.catalog.DeltaCatalog') \\  \n    .getOrCreate()  \n  shared_df = spark.read.format(\"deltaSharing\").load(table_url).select(\"column1\", \"column2\", \"column3\").show()\n\n","lang":"python"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["This requires that the Delta Sharing library has been installed and imported."]}]},"headings":[{"value":"Using the Delta Sharing Capability in a Custom Script Environment","id":"using-the-delta-sharing-capability-in-a-custom-script-environment","depth":1}],"frontmatter":{"seo":{"title":"Using the Delta Sharing Capability in a Custom Script Environment"}},"lastModified":"2026-03-09T03:12:50.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/products/customer-data-platform/integration-hub/zero-copy/using-the-delta-sharing-capability-in-a-custom-script-environment","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}