diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 61925e0cd..159d70f9f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -76,7 +76,7 @@ jobs: - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} run: | hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup - hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration + DATABRICKS_UNIQUE_ID="${{github.run_id}}_${{matrix.python-version}}_${{ matrix.airflow-version }}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 @@ -87,3 +87,47 @@ jobs: AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} + DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} + DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} + + Run-Integration-Tests-Expensive: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10"] + airflow-version: ["2.6"] + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install packages and dependencies + run: | + python -m pip install hatch + hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze + + - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} + run: | + hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup + DATABRICKS_UNIQUE_ID="${{github.run_id}}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-expensive + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + env: + AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ + AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH + AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} + DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} + DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index a66327d0e..29af4f731 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -2,6 +2,7 @@ import itertools import json import logging +import os from dataclasses import dataclass, field from subprocess import Popen, PIPE from typing import Any @@ -128,7 +129,9 @@ def load_via_dbt_ls(self): command.extend(["--select", *self.select]) logger.info(f"Running command: {command}") try: - process = Popen(command, stdout=PIPE, stderr=PIPE, cwd=self.project.dir, universal_newlines=True) + process = Popen( + command, stdout=PIPE, stderr=PIPE, cwd=self.project.dir, universal_newlines=True, env=os.environ + ) except FileNotFoundError as exception: raise CosmosLoadDbtException(f"Unable to run the command due to the error:\n{exception}") diff --git a/cosmos/dbt/parser/project.py b/cosmos/dbt/parser/project.py index cd7f20ac3..a390308e2 100644 --- a/cosmos/dbt/parser/project.py +++ b/cosmos/dbt/parser/project.py @@ -5,6 +5,8 @@ import logging import os +import ast + from dataclasses import dataclass, field from enum import Enum from pathlib import Path @@ -16,6 +18,11 @@ logger = logging.getLogger(__name__) +DBT_PY_MODEL_METHOD_NAME = "model" +DBT_PY_DEP_METHOD_NAME = "ref" +PYTHON_FILE_SUFFIX = ".py" + + class DbtModelType(Enum): """ Represents type of dbt unit (model, snapshot, seed) @@ -87,6 +94,30 @@ def _config_selector_ooo( return sql_configs +def extract_python_file_upstream_requirements(code: str) -> list[str]: + """ + Given a dbt Python model source code, identify other dbt entities which are upstream requirements. + + This method tries to find a `model` function and `dbt.ref` calls within it. Return a list of upstream entities IDs. + """ + source_code = ast.parse(code) + + upstream_entities = [] + model_function = "" + for node in source_code.body: + if isinstance(node, ast.FunctionDef) and node.name == DBT_PY_MODEL_METHOD_NAME: + model_function = node + break + + for item in ast.walk(model_function): + if isinstance(item, ast.Call) and item.func.attr == DBT_PY_DEP_METHOD_NAME: + upstream_entity_id = hasattr(item.args[-1], "value") and item.args[-1].value + if upstream_entity_id: + upstream_entities.append(upstream_entity_id) + + return upstream_entities + + @dataclass class DbtModel: """ @@ -124,30 +155,32 @@ def __post_init__(self) -> None: elif self.type == DbtModelType.DBT_SEED: code = None - # get the dependencies - env = jinja2.Environment() - ast = env.parse(code) - - # iterate over the jinja nodes to extract info - for base_node in ast.find_all(jinja2.nodes.Call): - if hasattr(base_node.node, "name"): - # check we have a ref - this indicates a dependency - if base_node.node.name == "ref": - # if it is, get the first argument - first_arg = base_node.args[0] - if isinstance(first_arg, jinja2.nodes.Const): - # and add it to the config - config.upstream_models.add(first_arg.value) - - # check if we have a config - this could contain tags - if base_node.node.name == "config": - # if it is, check if any kwargs are tags - for kwarg in base_node.kwargs: - for selector in self.config.config_types: - extracted_config = self._extract_config(kwarg=kwarg, config_name=selector) - config.config_selectors |= ( - set(extracted_config) if isinstance(extracted_config, (str, List)) else set() - ) + if self.path.suffix == PYTHON_FILE_SUFFIX: + config.upstream_models = config.upstream_models.union(set(extract_python_file_upstream_requirements(code))) + else: + # get the dependencies + env = jinja2.Environment() + jinja2_ast = env.parse(code) + # iterate over the jinja nodes to extract info + for base_node in jinja2_ast.find_all(jinja2.nodes.Call): + if hasattr(base_node.node, "name"): + # check we have a ref - this indicates a dependency + if base_node.node.name == "ref": + # if it is, get the first argument + first_arg = base_node.args[0] + if isinstance(first_arg, jinja2.nodes.Const): + # and add it to the config + config.upstream_models.add(first_arg.value) + + # check if we have a config - this could contain tags + if base_node.node.name == "config": + # if it is, check if any kwargs are tags + for kwarg in base_node.kwargs: + for selector in self.config.config_types: + extracted_config = self._extract_config(kwarg=kwarg, config_name=selector) + config.config_selectors |= ( + set(extracted_config) if isinstance(extracted_config, (str, List)) else set() + ) # set the config and set the parsed file flag to true self.config = config @@ -216,6 +249,10 @@ def __post_init__(self) -> None: for file_name in self.models_dir.rglob("*.sql"): self._handle_sql_file(file_name) + # crawl the models in the project + for file_name in self.models_dir.rglob("*.py"): + self._handle_sql_file(file_name) + # crawl the snapshots in the project for file_name in self.snapshots_dir.rglob("*.sql"): self._handle_sql_file(file_name) diff --git a/dev/dags/dbt/jaffle_shop_python/LICENSE b/dev/dags/dbt/jaffle_shop_python/LICENSE new file mode 100644 index 000000000..8dada3eda --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dev/dags/dbt/jaffle_shop_python/README.md b/dev/dags/dbt/jaffle_shop_python/README.md new file mode 100644 index 000000000..2ec873557 --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/README.md @@ -0,0 +1,15 @@ +## `jaffle_shop` + +`jaffle_shop` is a fictional ecommerce store. This dbt project transforms raw data from an app database into a customers and orders model ready for analytics. + +See [dbt's documentation](https://github.com/dbt-labs/jaffle_shop) for more info. + +### Modifications + +This project has been modified from the original to highlight some of the features of Cosmos. Namely: + +- tags have been added to the models +- the following models were replaced: + - `models/orders.sql` by `models/orders.py` + - `models/customers.sql` by `models/customers.py` +- the `profiles.yml` to interface with Databricks diff --git a/dev/dags/dbt/jaffle_shop_python/dbt_project.yml b/dev/dags/dbt/jaffle_shop_python/dbt_project.yml new file mode 100644 index 000000000..756faa2c4 --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/dbt_project.yml @@ -0,0 +1,20 @@ +name: 'jaffle_shop' + +config-version: 2 +version: '0.1' + +profile: 'jaffle_shop' + +model-paths: ["models"] +seed-paths: ["seeds"] +test-paths: ["tests"] +analysis-paths: ["analysis"] +macro-paths: ["macros"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_modules" + - "logs" + +require-dbt-version: [">=1.0.0", "<2.0.0"] diff --git a/dev/dags/dbt/jaffle_shop_python/macros/drop_table.sql b/dev/dags/dbt/jaffle_shop_python/macros/drop_table.sql new file mode 100644 index 000000000..ebd19e2d3 --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/macros/drop_table.sql @@ -0,0 +1,6 @@ +{%- macro drop_table(table_name) -%} + {%- set drop_query -%} + DROP TABLE IF EXISTS {{ target.schema }}.{{ table_name }} CASCADE + {%- endset -%} + {% do run_query(drop_query) %} +{%- endmacro -%} diff --git a/dev/dags/dbt/jaffle_shop_python/models/customers.py b/dev/dags/dbt/jaffle_shop_python/models/customers.py new file mode 100644 index 000000000..d3daa3bea --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/models/customers.py @@ -0,0 +1,44 @@ +import pyspark.sql.functions as F + + +def model(dbt, session): + stg_customers_df = dbt.ref("stg_customers") + stg_orders_df = dbt.ref("stg_orders") + stg_payments_df = dbt.ref("stg_payments") + + customer_orders_df = stg_orders_df.groupby("customer_id").agg( + F.min(F.col("order_date")).alias("first_order"), + F.max(F.col("order_date")).alias("most_recent_order"), + F.count(F.col("order_id")).alias("number_of_orders"), + ) + + customer_payments_df = ( + stg_payments_df.join(stg_orders_df, stg_payments_df.order_id == stg_orders_df.order_id, "left") + .groupby(stg_orders_df.customer_id) + .agg(F.sum(F.col("amount")).alias("total_amount")) + ) + + final_df = ( + stg_customers_df.alias("customers") + .join( + customer_orders_df.alias("customer_orders"), + F.col("customers.customer_id") == F.col("customer_orders.customer_id"), + "left", + ) + .join( + customer_payments_df.alias("customer_payments"), + F.col("customers.customer_id") == F.col("customer_payments.customer_id"), + "left", + ) + .select( + F.col("customers.customer_id").alias("customer_id"), + F.col("customers.first_name").alias("first_name"), + F.col("customers.last_name").alias("last_name"), + F.col("customer_orders.first_order").alias("first_order"), + F.col("customer_orders.most_recent_order").alias("most_recent_order"), + F.col("customer_orders.number_of_orders").alias("number_of_orders"), + F.col("customer_payments.total_amount").alias("customer_lifetime_value"), + ) + ) + + return final_df diff --git a/dev/dags/dbt/jaffle_shop_python/models/docs.md b/dev/dags/dbt/jaffle_shop_python/models/docs.md new file mode 100644 index 000000000..c6ae93be0 --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/models/docs.md @@ -0,0 +1,14 @@ +{% docs orders_status %} + +Orders can be one of the following statuses: + +| status | description | +|----------------|------------------------------------------------------------------------------------------------------------------------| +| placed | The order has been placed but has not yet left the warehouse | +| shipped | The order has ben shipped to the customer and is currently in transit | +| completed | The order has been received by the customer | +| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse | +| returned | The order has been returned by the customer and received at the warehouse | + + +{% enddocs %} diff --git a/dev/dags/dbt/jaffle_shop_python/models/orders.py b/dev/dags/dbt/jaffle_shop_python/models/orders.py new file mode 100644 index 000000000..64fdd03d9 --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/models/orders.py @@ -0,0 +1,31 @@ +import pyspark.sql.functions as F + + +def model(dbt, session): + stg_orders_df = dbt.ref("stg_orders") + stg_payments_df = dbt.ref("stg_payments") + + payment_methods = ["credit_card", "coupon", "bank_transfer", "gift_card"] + + agg_list = [ + F.sum(F.when(stg_payments_df.payment_method == payment_method, stg_payments_df.amount).otherwise(0)).alias( + payment_method + "_amount" + ) + for payment_method in payment_methods + ] + agg_list.append(F.sum(F.col("amount")).alias("total_amount")) + + order_payments_df = stg_payments_df.groupby("order_id").agg(*agg_list) + + final_df = stg_orders_df.join( + order_payments_df, stg_orders_df.order_id == order_payments_df.order_id, "left" + ).select( + stg_orders_df.order_id.alias("order_id"), + stg_orders_df.customer_id.alias("customer_id"), + stg_orders_df.order_date.alias("order_date"), + stg_orders_df.status.alias("status"), + *[F.col(payment_method + "_amount") for payment_method in payment_methods], + order_payments_df.total_amount.alias("amount"), + ) + + return final_df diff --git a/dev/dags/dbt/jaffle_shop_python/models/overview.md b/dev/dags/dbt/jaffle_shop_python/models/overview.md new file mode 100644 index 000000000..0544c42b1 --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/models/overview.md @@ -0,0 +1,11 @@ +{% docs __overview__ %} + +## Data Documentation for Jaffle Shop + +`jaffle_shop` is a fictional ecommerce store. + +This [dbt](https://www.getdbt.com/) project is for testing out code. + +The source code can be found [here](https://github.com/clrcrl/jaffle_shop). + +{% enddocs %} diff --git a/dev/dags/dbt/jaffle_shop_python/models/schema.yml b/dev/dags/dbt/jaffle_shop_python/models/schema.yml new file mode 100644 index 000000000..19e276707 --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/models/schema.yml @@ -0,0 +1,92 @@ +version: 2 + +models: + - name: customers + description: This table has basic information about a customer, as well as some derived facts based on a customer's orders + + columns: + - name: customer_id + description: This is a unique identifier for a customer + tests: + - unique + - not_null + + - name: first_name + description: Customer's first name. PII. + + - name: last_name + description: Customer's last name. PII. + + - name: first_order + description: Date (UTC) of a customer's first order + + - name: most_recent_order + description: Date (UTC) of a customer's most recent order + + - name: number_of_orders + description: Count of the number of orders a customer has placed + + - name: total_order_amount + description: Total value (AUD) of a customer's orders + + config: + cluster_id: "{{ env_var('DATABRICKS_CLUSTER_ID') }}" + submission_method: all_purpose_cluster + create_notebook: False + + - name: orders + description: This table has basic information about orders, as well as some derived facts based on payments + + columns: + - name: order_id + tests: + - unique + - not_null + description: This is a unique identifier for an order + + - name: customer_id + description: Foreign key to the customers table + tests: + - not_null + - relationships: + to: ref('customers') + field: customer_id + + - name: order_date + description: Date (UTC) that the order was placed + + - name: status + description: '{{ doc("orders_status") }}' + tests: + - accepted_values: + values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] + + - name: amount + description: Total amount (AUD) of the order + tests: + - not_null + + - name: credit_card_amount + description: Amount of the order (AUD) paid for by credit card + tests: + - not_null + + - name: coupon_amount + description: Amount of the order (AUD) paid for by coupon + tests: + - not_null + + - name: bank_transfer_amount + description: Amount of the order (AUD) paid for by bank transfer + tests: + - not_null + + - name: gift_card_amount + description: Amount of the order (AUD) paid for by gift card + tests: + - not_null + + config: + cluster_id: "{{ env_var('DATABRICKS_CLUSTER_ID') }}" + submission_method: all_purpose_cluster + create_notebook: False diff --git a/dev/dags/dbt/jaffle_shop_python/models/staging/schema.yml b/dev/dags/dbt/jaffle_shop_python/models/staging/schema.yml new file mode 100644 index 000000000..c207e4cf5 --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/models/staging/schema.yml @@ -0,0 +1,31 @@ +version: 2 + +models: + - name: stg_customers + columns: + - name: customer_id + tests: + - unique + - not_null + + - name: stg_orders + columns: + - name: order_id + tests: + - unique + - not_null + - name: status + tests: + - accepted_values: + values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] + + - name: stg_payments + columns: + - name: payment_id + tests: + - unique + - not_null + - name: payment_method + tests: + - accepted_values: + values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] diff --git a/dev/dags/dbt/jaffle_shop_python/models/staging/stg_customers.sql b/dev/dags/dbt/jaffle_shop_python/models/staging/stg_customers.sql new file mode 100644 index 000000000..cad047269 --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/models/staging/stg_customers.sql @@ -0,0 +1,22 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ ref('raw_customers') }} + +), + +renamed as ( + + select + id as customer_id, + first_name, + last_name + + from source + +) + +select * from renamed diff --git a/dev/dags/dbt/jaffle_shop_python/models/staging/stg_orders.sql b/dev/dags/dbt/jaffle_shop_python/models/staging/stg_orders.sql new file mode 100644 index 000000000..a654dcb94 --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/models/staging/stg_orders.sql @@ -0,0 +1,23 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ ref('raw_orders') }} + +), + +renamed as ( + + select + id as order_id, + user_id as customer_id, + order_date, + status + + from source + +) + +select * from renamed diff --git a/dev/dags/dbt/jaffle_shop_python/models/staging/stg_payments.sql b/dev/dags/dbt/jaffle_shop_python/models/staging/stg_payments.sql new file mode 100644 index 000000000..f718596ad --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/models/staging/stg_payments.sql @@ -0,0 +1,25 @@ +with source as ( + + {#- + Normally we would select from the table here, but we are using seeds to load + our data in this project + #} + select * from {{ ref('raw_payments') }} + +), + +renamed as ( + + select + id as payment_id, + order_id, + payment_method, + + -- `amount` is currently stored in cents, so we convert it to dollars + amount / 100 as amount + + from source + +) + +select * from renamed diff --git a/dev/dags/dbt/jaffle_shop_python/profiles.yml b/dev/dags/dbt/jaffle_shop_python/profiles.yml new file mode 100644 index 000000000..38282cc8f --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/profiles.yml @@ -0,0 +1,11 @@ +jaffle_shop: + outputs: + dev: + catalog: null + host: "{{ env_var('DATABRICKS_HOST') }}" + http_path: /sql/1.0/warehouses/{{ env_var('DATABRICKS_WAREHOUSE_ID') }} + schema: cosmos + threads: 1 + token: "{{ env_var('DATABRICKS_TOKEN') }}" + type: databricks + target: dev diff --git a/dev/dags/dbt/jaffle_shop_python/seeds/raw_customers.csv b/dev/dags/dbt/jaffle_shop_python/seeds/raw_customers.csv new file mode 100644 index 000000000..b3e6747d6 --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/seeds/raw_customers.csv @@ -0,0 +1,101 @@ +id,first_name,last_name +1,Michael,P. +2,Shawn,M. +3,Kathleen,P. +4,Jimmy,C. +5,Katherine,R. +6,Sarah,R. +7,Martin,M. +8,Frank,R. +9,Jennifer,F. +10,Henry,W. +11,Fred,S. +12,Amy,D. +13,Kathleen,M. +14,Steve,F. +15,Teresa,H. +16,Amanda,H. +17,Kimberly,R. +18,Johnny,K. +19,Virginia,F. +20,Anna,A. +21,Willie,H. +22,Sean,H. +23,Mildred,A. +24,David,G. +25,Victor,H. +26,Aaron,R. +27,Benjamin,B. +28,Lisa,W. +29,Benjamin,K. +30,Christina,W. +31,Jane,G. +32,Thomas,O. +33,Katherine,M. +34,Jennifer,S. +35,Sara,T. +36,Harold,O. +37,Shirley,J. +38,Dennis,J. +39,Louise,W. +40,Maria,A. +41,Gloria,C. +42,Diana,S. +43,Kelly,N. +44,Jane,R. +45,Scott,B. +46,Norma,C. +47,Marie,P. +48,Lillian,C. +49,Judy,N. +50,Billy,L. +51,Howard,R. +52,Laura,F. +53,Anne,B. +54,Rose,M. +55,Nicholas,R. +56,Joshua,K. +57,Paul,W. +58,Kathryn,K. +59,Adam,A. +60,Norma,W. +61,Timothy,R. +62,Elizabeth,P. +63,Edward,G. +64,David,C. +65,Brenda,W. +66,Adam,W. +67,Michael,H. +68,Jesse,E. +69,Janet,P. +70,Helen,F. +71,Gerald,C. +72,Kathryn,O. +73,Alan,B. +74,Harry,A. +75,Andrea,H. +76,Barbara,W. +77,Anne,W. +78,Harry,H. +79,Jack,R. +80,Phillip,H. +81,Shirley,H. +82,Arthur,D. +83,Virginia,R. +84,Christina,R. +85,Theresa,M. +86,Jason,C. +87,Phillip,B. +88,Adam,T. +89,Margaret,J. +90,Paul,P. +91,Todd,W. +92,Willie,O. +93,Frances,R. +94,Gregory,H. +95,Lisa,P. +96,Jacqueline,A. +97,Shirley,D. +98,Nicole,M. +99,Mary,G. +100,Jean,M. diff --git a/dev/dags/dbt/jaffle_shop_python/seeds/raw_orders.csv b/dev/dags/dbt/jaffle_shop_python/seeds/raw_orders.csv new file mode 100644 index 000000000..c4870621b --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/seeds/raw_orders.csv @@ -0,0 +1,100 @@ +id,user_id,order_date,status +1,1,2018-01-01,returned +2,3,2018-01-02,completed +3,94,2018-01-04,completed +4,50,2018-01-05,completed +5,64,2018-01-05,completed +6,54,2018-01-07,completed +7,88,2018-01-09,completed +8,2,2018-01-11,returned +9,53,2018-01-12,completed +10,7,2018-01-14,completed +11,99,2018-01-14,completed +12,59,2018-01-15,completed +13,84,2018-01-17,completed +14,40,2018-01-17,returned +15,25,2018-01-17,completed +16,39,2018-01-18,completed +17,71,2018-01-18,completed +18,64,2018-01-20,returned +19,54,2018-01-22,completed +20,20,2018-01-23,completed +21,71,2018-01-23,completed +22,86,2018-01-24,completed +23,22,2018-01-26,return_pending +24,3,2018-01-27,completed +25,51,2018-01-28,completed +26,32,2018-01-28,completed +27,94,2018-01-29,completed +28,8,2018-01-29,completed +29,57,2018-01-31,completed +30,69,2018-02-02,completed +31,16,2018-02-02,completed +32,28,2018-02-04,completed +33,42,2018-02-04,completed +34,38,2018-02-06,completed +35,80,2018-02-08,completed +36,85,2018-02-10,completed +37,1,2018-02-10,completed +38,51,2018-02-10,completed +39,26,2018-02-11,completed +40,33,2018-02-13,completed +41,99,2018-02-14,completed +42,92,2018-02-16,completed +43,31,2018-02-17,completed +44,66,2018-02-17,completed +45,22,2018-02-17,completed +46,6,2018-02-19,completed +47,50,2018-02-20,completed +48,27,2018-02-21,completed +49,35,2018-02-21,completed +50,51,2018-02-23,completed +51,71,2018-02-24,completed +52,54,2018-02-25,return_pending +53,34,2018-02-26,completed +54,54,2018-02-26,completed +55,18,2018-02-27,completed +56,79,2018-02-28,completed +57,93,2018-03-01,completed +58,22,2018-03-01,completed +59,30,2018-03-02,completed +60,12,2018-03-03,completed +61,63,2018-03-03,completed +62,57,2018-03-05,completed +63,70,2018-03-06,completed +64,13,2018-03-07,completed +65,26,2018-03-08,completed +66,36,2018-03-10,completed +67,79,2018-03-11,completed +68,53,2018-03-11,completed +69,3,2018-03-11,completed +70,8,2018-03-12,completed +71,42,2018-03-12,shipped +72,30,2018-03-14,shipped +73,19,2018-03-16,completed +74,9,2018-03-17,shipped +75,69,2018-03-18,completed +76,25,2018-03-20,completed +77,35,2018-03-21,shipped +78,90,2018-03-23,shipped +79,52,2018-03-23,shipped +80,11,2018-03-23,shipped +81,76,2018-03-23,shipped +82,46,2018-03-24,shipped +83,54,2018-03-24,shipped +84,70,2018-03-26,placed +85,47,2018-03-26,shipped +86,68,2018-03-26,placed +87,46,2018-03-27,placed +88,91,2018-03-27,shipped +89,21,2018-03-28,placed +90,66,2018-03-30,shipped +91,47,2018-03-31,placed +92,84,2018-04-02,placed +93,66,2018-04-03,placed +94,63,2018-04-03,placed +95,27,2018-04-04,placed +96,90,2018-04-06,placed +97,89,2018-04-07,placed +98,41,2018-04-07,placed +99,85,2018-04-09,placed diff --git a/dev/dags/dbt/jaffle_shop_python/seeds/raw_payments.csv b/dev/dags/dbt/jaffle_shop_python/seeds/raw_payments.csv new file mode 100644 index 000000000..a587baab5 --- /dev/null +++ b/dev/dags/dbt/jaffle_shop_python/seeds/raw_payments.csv @@ -0,0 +1,114 @@ +id,order_id,payment_method,amount +1,1,credit_card,1000 +2,2,credit_card,2000 +3,3,coupon,100 +4,4,coupon,2500 +5,5,bank_transfer,1700 +6,6,credit_card,600 +7,7,credit_card,1600 +8,8,credit_card,2300 +9,9,gift_card,2300 +10,9,bank_transfer,0 +11,10,bank_transfer,2600 +12,11,credit_card,2700 +13,12,credit_card,100 +14,13,credit_card,500 +15,13,bank_transfer,1400 +16,14,bank_transfer,300 +17,15,coupon,2200 +18,16,credit_card,1000 +19,17,bank_transfer,200 +20,18,credit_card,500 +21,18,credit_card,800 +22,19,gift_card,600 +23,20,bank_transfer,1500 +24,21,credit_card,1200 +25,22,bank_transfer,800 +26,23,gift_card,2300 +27,24,coupon,2600 +28,25,bank_transfer,2000 +29,25,credit_card,2200 +30,25,coupon,1600 +31,26,credit_card,3000 +32,27,credit_card,2300 +33,28,bank_transfer,1900 +34,29,bank_transfer,1200 +35,30,credit_card,1300 +36,31,credit_card,1200 +37,32,credit_card,300 +38,33,credit_card,2200 +39,34,bank_transfer,1500 +40,35,credit_card,2900 +41,36,bank_transfer,900 +42,37,credit_card,2300 +43,38,credit_card,1500 +44,39,bank_transfer,800 +45,40,credit_card,1400 +46,41,credit_card,1700 +47,42,coupon,1700 +48,43,gift_card,1800 +49,44,gift_card,1100 +50,45,bank_transfer,500 +51,46,bank_transfer,800 +52,47,credit_card,2200 +53,48,bank_transfer,300 +54,49,credit_card,600 +55,49,credit_card,900 +56,50,credit_card,2600 +57,51,credit_card,2900 +58,51,credit_card,100 +59,52,bank_transfer,1500 +60,53,credit_card,300 +61,54,credit_card,1800 +62,54,bank_transfer,1100 +63,55,credit_card,2900 +64,56,credit_card,400 +65,57,bank_transfer,200 +66,58,coupon,1800 +67,58,gift_card,600 +68,59,gift_card,2800 +69,60,credit_card,400 +70,61,bank_transfer,1600 +71,62,gift_card,1400 +72,63,credit_card,2900 +73,64,bank_transfer,2600 +74,65,credit_card,0 +75,66,credit_card,2800 +76,67,bank_transfer,400 +77,67,credit_card,1900 +78,68,credit_card,1600 +79,69,credit_card,1900 +80,70,credit_card,2600 +81,71,credit_card,500 +82,72,credit_card,2900 +83,73,bank_transfer,300 +84,74,credit_card,3000 +85,75,credit_card,1900 +86,76,coupon,200 +87,77,credit_card,0 +88,77,bank_transfer,1900 +89,78,bank_transfer,2600 +90,79,credit_card,1800 +91,79,credit_card,900 +92,80,gift_card,300 +93,81,coupon,200 +94,82,credit_card,800 +95,83,credit_card,100 +96,84,bank_transfer,2500 +97,85,bank_transfer,1700 +98,86,coupon,2300 +99,87,gift_card,3000 +100,87,credit_card,2600 +101,88,credit_card,2900 +102,89,bank_transfer,2200 +103,90,bank_transfer,200 +104,91,credit_card,1900 +105,92,bank_transfer,1500 +106,92,coupon,200 +107,93,gift_card,2600 +108,94,coupon,700 +109,95,coupon,2400 +110,96,gift_card,1700 +111,97,bank_transfer,1400 +112,98,bank_transfer,1000 +113,99,credit_card,2400 diff --git a/dev/dags/example_cosmos_python_models.py b/dev/dags/example_cosmos_python_models.py new file mode 100644 index 000000000..2876e38c2 --- /dev/null +++ b/dev/dags/example_cosmos_python_models.py @@ -0,0 +1,44 @@ +""" +An example DAG that uses Cosmos to render a dbt project which uses Python models (Databricks). + +It relies on the user setting the connection databricks_default. + +This can be done via environment variable: + +export AIRFLOW_CONN_DATABRICKS_DEFAULT=databricks://@dbc-.cloud.databricks.com?token=&http_path=/sql/1.0/warehouses/ + +Replacing: +- : Databricks account ID +- : Databricks access token +- : Databricks SQL Warehouse ID from connection HTTP path +""" + +import os +from datetime import datetime +from pathlib import Path + +from cosmos import DbtDag + +DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" +DBT_ROOT_PATH = Path(os.getenv("DBT_ROOT_PATH", DEFAULT_DBT_ROOT_PATH)) +SCHEMA = "cosmos_" + os.getenv("DATABRICKS_UNIQUE_ID", "").replace(".", "_") + +# [START example_cosmos_python_models] +example_cosmos_python_models = DbtDag( + # dbt/cosmos-specific parameters + dbt_root_path=DBT_ROOT_PATH, + dbt_project_name="jaffle_shop_python", + conn_id="databricks_default", + profile_args={ + "schema": SCHEMA, + }, + operator_args={"append_env": True}, + profile_name_override="airflow", + target_name_override="dev_target", + # normal dag parameters + schedule_interval="@daily", + start_date=datetime(2023, 1, 1), + catchup=False, + dag_id="example_cosmos_python_models", +) +# [END example_cosmos_python_models] diff --git a/pyproject.toml b/pyproject.toml index 9c8701193..8f56514df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -156,8 +156,23 @@ test-cov = 'pytest -vv --cov=cosmos --cov-report=term-missing --cov-report=xml - test-integration-setup = """pip uninstall dbt-postgres; \ rm -rf airflow.*; \ airflow db init; \ -pip install dbt-postgres;""" -test-integration = """pytest -vv --cov=cosmos --cov-report=term-missing --cov-report=xml --durations=0 -m integration""" +pip install dbt-postgres; \ +pip install dbt-databricks""" +test-integration = """pytest -vv \ +--cov=cosmos \ +--cov-report=term-missing \ +--cov-report=xml \ +--durations=0 \ +-m integration \ +-k 'not test_example_dag[example_cosmos_python_models]' +""" +test-integration-expensive = """pytest -vv \ +--cov=cosmos \ +--cov-report=term-missing \ +--cov-report=xml \ +--durations=0 \ +-m integration \ +-k 'test_example_dag[example_cosmos_python_models]'""" [tool.pytest.ini_options] filterwarnings = [ @@ -165,7 +180,7 @@ filterwarnings = [ ] minversion = "6.0" markers = [ - "integration" + "integration", ] ###################################### diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 1585f5ea8..f1f852917 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -9,10 +9,15 @@ DBT_PROJECTS_ROOT_DIR = Path(__file__).parent.parent.parent / "dev/dags/dbt" SAMPLE_MANIFEST = Path(__file__).parent.parent / "sample/manifest.json" +SAMPLE_MANIFEST_PY = Path(__file__).parent.parent / "sample/manifest_python.json" -def test_load_via_manifest_with_exclude(): - dbt_project = DbtProject(name="jaffle_shop", root_dir=DBT_PROJECTS_ROOT_DIR, manifest_path=SAMPLE_MANIFEST) +@pytest.mark.parametrize( + "pipeline_name,manifest_filepath,model_filepath", + [("jaffle_shop", SAMPLE_MANIFEST, "customers.sql"), ("jaffle_shop_python", SAMPLE_MANIFEST_PY, "customers.py")], +) +def test_load_via_manifest_with_exclude(pipeline_name, manifest_filepath, model_filepath): + dbt_project = DbtProject(name=pipeline_name, root_dir=DBT_PROJECTS_ROOT_DIR, manifest_path=manifest_filepath) dbt_graph = DbtGraph(project=dbt_project, exclude=["config.materialized:table"]) dbt_graph.load_from_dbt_manifest() @@ -29,7 +34,7 @@ def test_load_via_manifest_with_exclude(): "model.jaffle_shop.stg_orders", "model.jaffle_shop.stg_payments", ] - assert sample_node.file_path == DBT_PROJECTS_ROOT_DIR / "jaffle_shop/models/customers.sql" + assert sample_node.file_path == DBT_PROJECTS_ROOT_DIR / f"{pipeline_name}/models/{model_filepath}" @patch("cosmos.dbt.graph.DbtGraph.load_from_dbt_manifest", return_value=None) @@ -132,8 +137,9 @@ def test_load_via_dbt_ls_with_exclude(): @pytest.mark.integration -def test_load_via_dbt_ls_without_exclude(): - dbt_project = DbtProject(name="jaffle_shop", root_dir=DBT_PROJECTS_ROOT_DIR) +@pytest.mark.parametrize("pipeline_name", ("jaffle_shop", "jaffle_shop_python")) +def test_load_via_dbt_ls_without_exclude(pipeline_name): + dbt_project = DbtProject(name=pipeline_name, root_dir=DBT_PROJECTS_ROOT_DIR) dbt_graph = DbtGraph(project=dbt_project) dbt_graph.load_via_dbt_ls() @@ -163,9 +169,10 @@ def test_load_via_dbt_ls_with_runtime_error_in_stdout(mock_popen_communicate): mock_popen_communicate.assert_called_once() -def test_load_via_load_via_custom_parser(): +@pytest.mark.parametrize("pipeline_name", ("jaffle_shop", "jaffle_shop_python")) +def test_load_via_load_via_custom_parser(pipeline_name): dbt_project = DbtProject( - name="jaffle_shop", + name=pipeline_name, root_dir=DBT_PROJECTS_ROOT_DIR, ) dbt_graph = DbtGraph(project=dbt_project) diff --git a/tests/sample/manifest_python.json b/tests/sample/manifest_python.json new file mode 100644 index 000000000..cd9b8fb78 --- /dev/null +++ b/tests/sample/manifest_python.json @@ -0,0 +1,12771 @@ +{ + "child_map": { + "model.jaffle_shop.customers": [ + "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d", + "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", + "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1" + ], + "model.jaffle_shop.orders": [ + "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3", + "test.jaffle_shop.not_null_orders_amount.106140f9fd", + "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49", + "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625", + "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59", + "test.jaffle_shop.not_null_orders_customer_id.c5f02694af", + "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a", + "test.jaffle_shop.not_null_orders_order_id.cf6c17daed", + "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", + "test.jaffle_shop.unique_orders_order_id.fed79b3a6e" + ], + "model.jaffle_shop.stg_customers": [ + "model.jaffle_shop.customers", + "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa", + "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada" + ], + "model.jaffle_shop.stg_orders": [ + "model.jaffle_shop.customers", + "model.jaffle_shop.orders", + "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad", + "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64", + "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a" + ], + "model.jaffle_shop.stg_payments": [ + "model.jaffle_shop.customers", + "model.jaffle_shop.orders", + "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278", + "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075", + "test.jaffle_shop.unique_stg_payments_payment_id.3744510712" + ], + "seed.jaffle_shop.raw_customers": [ + "model.jaffle_shop.stg_customers" + ], + "seed.jaffle_shop.raw_orders": [ + "model.jaffle_shop.stg_orders" + ], + "seed.jaffle_shop.raw_payments": [ + "model.jaffle_shop.stg_payments" + ], + "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3": [], + "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad": [], + "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278": [], + "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d": [], + "test.jaffle_shop.not_null_orders_amount.106140f9fd": [], + "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49": [], + "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625": [], + "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59": [], + "test.jaffle_shop.not_null_orders_customer_id.c5f02694af": [], + "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a": [], + "test.jaffle_shop.not_null_orders_order_id.cf6c17daed": [], + "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa": [], + "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64": [], + "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075": [], + "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2": [], + "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1": [], + "test.jaffle_shop.unique_orders_order_id.fed79b3a6e": [], + "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada": [], + "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a": [], + "test.jaffle_shop.unique_stg_payments_payment_id.3744510712": [] + }, + "disabled": {}, + "docs": { + "doc.dbt.__overview__": { + "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion", + "name": "__overview__", + "original_file_path": "docs/overview.md", + "package_name": "dbt", + "path": "overview.md", + "resource_type": "doc", + "unique_id": "doc.dbt.__overview__" + }, + "doc.jaffle_shop.__overview__": { + "block_contents": "## Data Documentation for Jaffle Shop\n\n`jaffle_shop` is a fictional ecommerce store.\n\nThis [dbt](https://www.getdbt.com/) project is for testing out code.\n\nThe source code can be found [here](https://github.com/clrcrl/jaffle_shop).", + "name": "__overview__", + "original_file_path": "models/overview.md", + "package_name": "jaffle_shop", + "path": "overview.md", + "resource_type": "doc", + "unique_id": "doc.jaffle_shop.__overview__" + }, + "doc.jaffle_shop.orders_status": { + "block_contents": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", + "name": "orders_status", + "original_file_path": "models/docs.md", + "package_name": "jaffle_shop", + "path": "docs.md", + "resource_type": "doc", + "unique_id": "doc.jaffle_shop.orders_status" + } + }, + "exposures": {}, + "group_map": {}, + "groups": {}, + "macros": { + "macro.dbt._split_part_negative": { + "arguments": [], + "created_at": 1689766376.5609128, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 {{ part_number }}\n )\n\n{% endmacro %}", + "meta": {}, + "name": "_split_part_negative", + "original_file_path": "macros/utils/split_part.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/split_part.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt._split_part_negative" + }, + "macro.dbt.after_commit": { + "arguments": [], + "created_at": 1689766376.446445, + "depends_on": { + "macros": [ + "macro.dbt.make_hook_config" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", + "meta": {}, + "name": "after_commit", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.after_commit" + }, + "macro.dbt.alter_column_comment": { + "arguments": [], + "created_at": 1689766376.582122, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__alter_column_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_column_comment", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.alter_column_comment" + }, + "macro.dbt.alter_column_type": { + "arguments": [], + "created_at": 1689766376.592001, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__alter_column_type" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_column_type", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.alter_column_type" + }, + "macro.dbt.alter_relation_add_remove_columns": { + "arguments": [], + "created_at": 1689766376.592896, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__alter_relation_add_remove_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_relation_add_remove_columns", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.alter_relation_add_remove_columns" + }, + "macro.dbt.alter_relation_comment": { + "arguments": [], + "created_at": 1689766376.5825138, + "depends_on": { + "macros": [ + "macro.dbt.default__alter_relation_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_relation_comment", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.alter_relation_comment" + }, + "macro.dbt.any_value": { + "arguments": [], + "created_at": 1689766376.554431, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__any_value" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "any_value", + "original_file_path": "macros/utils/any_value.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/any_value.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.any_value" + }, + "macro.dbt.apply_grants": { + "arguments": [], + "created_at": 1689766376.580053, + "depends_on": { + "macros": [ + "macro.dbt.default__apply_grants" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", + "meta": {}, + "name": "apply_grants", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.apply_grants" + }, + "macro.dbt.array_append": { + "arguments": [], + "created_at": 1689766376.562217, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__array_append" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", + "meta": {}, + "name": "array_append", + "original_file_path": "macros/utils/array_append.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_append.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.array_append" + }, + "macro.dbt.array_concat": { + "arguments": [], + "created_at": 1689766376.558669, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__array_concat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", + "meta": {}, + "name": "array_concat", + "original_file_path": "macros/utils/array_concat.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_concat.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.array_concat" + }, + "macro.dbt.array_construct": { + "arguments": [], + "created_at": 1689766376.561696, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__array_construct" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", + "meta": {}, + "name": "array_construct", + "original_file_path": "macros/utils/array_construct.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_construct.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.array_construct" + }, + "macro.dbt.assert_columns_equivalent": { + "arguments": [], + "created_at": 1689766376.511669, + "depends_on": { + "macros": [ + "macro.dbt.get_column_schema_from_query", + "macro.dbt.get_empty_schema_sql", + "macro.dbt.format_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(model['columns'])) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", + "meta": {}, + "name": "assert_columns_equivalent", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.assert_columns_equivalent" + }, + "macro.dbt.before_begin": { + "arguments": [], + "created_at": 1689766376.4461498, + "depends_on": { + "macros": [ + "macro.dbt.make_hook_config" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", + "meta": {}, + "name": "before_begin", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.before_begin" + }, + "macro.dbt.bool_or": { + "arguments": [], + "created_at": 1689766376.559051, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__bool_or" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "bool_or", + "original_file_path": "macros/utils/bool_or.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/bool_or.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.bool_or" + }, + "macro.dbt.build_config_dict": { + "arguments": [], + "created_at": 1689766376.597595, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", + "meta": {}, + "name": "build_config_dict", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_config_dict" + }, + "macro.dbt.build_ref_function": { + "arguments": [], + "created_at": 1689766376.596475, + "depends_on": { + "macros": [ + "macro.dbt.resolve_model_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", + "meta": {}, + "name": "build_ref_function", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_ref_function" + }, + "macro.dbt.build_snapshot_staging_table": { + "arguments": [], + "created_at": 1689766376.464192, + "depends_on": { + "macros": [ + "macro.dbt.make_temp_relation", + "macro.dbt.snapshot_staging_table", + "macro.dbt.statement", + "macro.dbt.create_table_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", + "meta": {}, + "name": "build_snapshot_staging_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_snapshot_staging_table" + }, + "macro.dbt.build_snapshot_table": { + "arguments": [], + "created_at": 1689766376.463488, + "depends_on": { + "macros": [ + "macro.dbt.default__build_snapshot_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", + "meta": {}, + "name": "build_snapshot_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_snapshot_table" + }, + "macro.dbt.build_source_function": { + "arguments": [], + "created_at": 1689766376.5969868, + "depends_on": { + "macros": [ + "macro.dbt.resolve_model_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", + "meta": {}, + "name": "build_source_function", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_source_function" + }, + "macro.dbt.call_dcl_statements": { + "arguments": [], + "created_at": 1689766376.57935, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__call_dcl_statements" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", + "meta": {}, + "name": "call_dcl_statements", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.call_dcl_statements" + }, + "macro.dbt.cast_bool_to_text": { + "arguments": [], + "created_at": 1689766376.5540211, + "depends_on": { + "macros": [ + "macro.dbt.default__cast_bool_to_text" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", + "meta": {}, + "name": "cast_bool_to_text", + "original_file_path": "macros/utils/cast_bool_to_text.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/cast_bool_to_text.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.cast_bool_to_text" + }, + "macro.dbt.check_for_schema_changes": { + "arguments": [], + "created_at": 1689766376.506442, + "depends_on": { + "macros": [ + "macro.dbt.diff_columns", + "macro.dbt.diff_column_data_types" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", + "meta": {}, + "name": "check_for_schema_changes", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.check_for_schema_changes" + }, + "macro.dbt.check_schema_exists": { + "arguments": [], + "created_at": 1689766376.586348, + "depends_on": { + "macros": [ + "macro.dbt.default__check_schema_exists" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", + "meta": {}, + "name": "check_schema_exists", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.check_schema_exists" + }, + "macro.dbt.collect_freshness": { + "arguments": [], + "created_at": 1689766376.573952, + "depends_on": { + "macros": [ + "macro.dbt.default__collect_freshness" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", + "meta": {}, + "name": "collect_freshness", + "original_file_path": "macros/adapters/freshness.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/freshness.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.collect_freshness" + }, + "macro.dbt.concat": { + "arguments": [], + "created_at": 1689766376.5491068, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__concat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", + "meta": {}, + "name": "concat", + "original_file_path": "macros/utils/concat.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/concat.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.concat" + }, + "macro.dbt.convert_datetime": { + "arguments": [], + "created_at": 1689766376.545648, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", + "meta": {}, + "name": "convert_datetime", + "original_file_path": "macros/etc/datetime.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/datetime.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.convert_datetime" + }, + "macro.dbt.copy_grants": { + "arguments": [], + "created_at": 1689766376.576125, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__copy_grants" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "copy_grants", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.copy_grants" + }, + "macro.dbt.create_columns": { + "arguments": [], + "created_at": 1689766376.461233, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__create_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", + "meta": {}, + "name": "create_columns", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_columns" + }, + "macro.dbt.create_csv_table": { + "arguments": [], + "created_at": 1689766376.532575, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__create_csv_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", + "meta": {}, + "name": "create_csv_table", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_csv_table" + }, + "macro.dbt.create_indexes": { + "arguments": [], + "created_at": 1689766376.565547, + "depends_on": { + "macros": [ + "macro.dbt.default__create_indexes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", + "meta": {}, + "name": "create_indexes", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_indexes" + }, + "macro.dbt.create_or_replace_view": { + "arguments": [], + "created_at": 1689766376.522515, + "depends_on": { + "macros": [ + "macro.dbt.run_hooks", + "macro.dbt.handle_existing_table", + "macro.dbt.should_full_refresh", + "macro.dbt.statement", + "macro.dbt.get_create_view_as_sql", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", + "meta": {}, + "name": "create_or_replace_view", + "original_file_path": "macros/materializations/models/view/create_or_replace_view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/create_or_replace_view.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_or_replace_view" + }, + "macro.dbt.create_schema": { + "arguments": [], + "created_at": 1689766376.5627258, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__create_schema" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", + "meta": {}, + "name": "create_schema", + "original_file_path": "macros/adapters/schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_schema" + }, + "macro.dbt.create_table_as": { + "arguments": [], + "created_at": 1689766376.516692, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__create_table_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", + "meta": {}, + "name": "create_table_as", + "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/create_table_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_table_as" + }, + "macro.dbt.create_view_as": { + "arguments": [], + "created_at": 1689766376.523294, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__create_view_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", + "meta": {}, + "name": "create_view_as", + "original_file_path": "macros/materializations/models/view/create_view_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/create_view_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_view_as" + }, + "macro.dbt.current_timestamp": { + "arguments": [], + "created_at": 1689766376.563742, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__current_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "current_timestamp", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.current_timestamp" + }, + "macro.dbt.current_timestamp_backcompat": { + "arguments": [], + "created_at": 1689766376.56431, + "depends_on": { + "macros": [ + "macro.dbt.default__current_timestamp_backcompat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "current_timestamp_backcompat", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.current_timestamp_backcompat" + }, + "macro.dbt.current_timestamp_in_utc_backcompat": { + "arguments": [], + "created_at": 1689766376.564553, + "depends_on": { + "macros": [ + "macro.dbt.default__current_timestamp_in_utc_backcompat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "current_timestamp_in_utc_backcompat", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat" + }, + "macro.dbt.date_trunc": { + "arguments": [], + "created_at": 1689766376.5611892, + "depends_on": { + "macros": [ + "macro.dbt.default__date_trunc" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", + "meta": {}, + "name": "date_trunc", + "original_file_path": "macros/utils/date_trunc.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date_trunc.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.date_trunc" + }, + "macro.dbt.dateadd": { + "arguments": [], + "created_at": 1689766376.5499609, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__dateadd" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", + "meta": {}, + "name": "dateadd", + "original_file_path": "macros/utils/dateadd.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/dateadd.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.dateadd" + }, + "macro.dbt.datediff": { + "arguments": [], + "created_at": 1689766376.552764, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__datediff" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", + "meta": {}, + "name": "datediff", + "original_file_path": "macros/utils/datediff.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/datediff.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.datediff" + }, + "macro.dbt.dates_in_range": { + "arguments": [], + "created_at": 1689766376.546978, + "depends_on": { + "macros": [ + "macro.dbt.convert_datetime" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", + "meta": {}, + "name": "dates_in_range", + "original_file_path": "macros/etc/datetime.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/datetime.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.dates_in_range" + }, + "macro.dbt.default__alter_column_comment": { + "arguments": [], + "created_at": 1689766376.582299, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__alter_column_comment", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__alter_column_comment" + }, + "macro.dbt.default__alter_column_type": { + "arguments": [], + "created_at": 1689766376.59263, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__alter_column_type", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__alter_column_type" + }, + "macro.dbt.default__alter_relation_add_remove_columns": { + "arguments": [], + "created_at": 1689766376.5936918, + "depends_on": { + "macros": [ + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__alter_relation_add_remove_columns", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__alter_relation_add_remove_columns" + }, + "macro.dbt.default__alter_relation_comment": { + "arguments": [], + "created_at": 1689766376.58269, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__alter_relation_comment", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__alter_relation_comment" + }, + "macro.dbt.default__any_value": { + "arguments": [], + "created_at": 1689766376.554538, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", + "meta": {}, + "name": "default__any_value", + "original_file_path": "macros/utils/any_value.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/any_value.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__any_value" + }, + "macro.dbt.default__apply_grants": { + "arguments": [], + "created_at": 1689766376.58136, + "depends_on": { + "macros": [ + "macro.dbt.run_query", + "macro.dbt.get_show_grant_sql", + "macro.dbt.get_dcl_statement_list", + "macro.dbt.call_dcl_statements" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "default__apply_grants", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__apply_grants" + }, + "macro.dbt.default__array_append": { + "arguments": [], + "created_at": 1689766376.562351, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", + "meta": {}, + "name": "default__array_append", + "original_file_path": "macros/utils/array_append.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_append.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__array_append" + }, + "macro.dbt.default__array_concat": { + "arguments": [], + "created_at": 1689766376.5588028, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", + "meta": {}, + "name": "default__array_concat", + "original_file_path": "macros/utils/array_concat.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_concat.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__array_concat" + }, + "macro.dbt.default__array_construct": { + "arguments": [], + "created_at": 1689766376.5619369, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", + "meta": {}, + "name": "default__array_construct", + "original_file_path": "macros/utils/array_construct.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_construct.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__array_construct" + }, + "macro.dbt.default__bool_or": { + "arguments": [], + "created_at": 1689766376.5591521, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", + "meta": {}, + "name": "default__bool_or", + "original_file_path": "macros/utils/bool_or.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/bool_or.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__bool_or" + }, + "macro.dbt.default__build_snapshot_table": { + "arguments": [], + "created_at": 1689766376.4637492, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", + "meta": {}, + "name": "default__build_snapshot_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__build_snapshot_table" + }, + "macro.dbt.default__call_dcl_statements": { + "arguments": [], + "created_at": 1689766376.579675, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__call_dcl_statements", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__call_dcl_statements" + }, + "macro.dbt.default__cast_bool_to_text": { + "arguments": [], + "created_at": 1689766376.554183, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", + "meta": {}, + "name": "default__cast_bool_to_text", + "original_file_path": "macros/utils/cast_bool_to_text.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/cast_bool_to_text.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__cast_bool_to_text" + }, + "macro.dbt.default__check_schema_exists": { + "arguments": [], + "created_at": 1689766376.5866601, + "depends_on": { + "macros": [ + "macro.dbt.replace", + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__check_schema_exists", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__check_schema_exists" + }, + "macro.dbt.default__collect_freshness": { + "arguments": [], + "created_at": 1689766376.574364, + "depends_on": { + "macros": [ + "macro.dbt.statement", + "macro.dbt.current_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", + "meta": {}, + "name": "default__collect_freshness", + "original_file_path": "macros/adapters/freshness.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/freshness.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__collect_freshness" + }, + "macro.dbt.default__concat": { + "arguments": [], + "created_at": 1689766376.549238, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", + "meta": {}, + "name": "default__concat", + "original_file_path": "macros/utils/concat.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/concat.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__concat" + }, + "macro.dbt.default__copy_grants": { + "arguments": [], + "created_at": 1689766376.5762491, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", + "meta": {}, + "name": "default__copy_grants", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__copy_grants" + }, + "macro.dbt.default__create_columns": { + "arguments": [], + "created_at": 1689766376.4615622, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "default__create_columns", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_columns" + }, + "macro.dbt.default__create_csv_table": { + "arguments": [], + "created_at": 1689766376.5334961, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", + "meta": {}, + "name": "default__create_csv_table", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_csv_table" + }, + "macro.dbt.default__create_indexes": { + "arguments": [], + "created_at": 1689766376.56596, + "depends_on": { + "macros": [ + "macro.dbt.get_create_index_sql", + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "default__create_indexes", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_indexes" + }, + "macro.dbt.default__create_schema": { + "arguments": [], + "created_at": 1689766376.562908, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__create_schema", + "original_file_path": "macros/adapters/schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_schema" + }, + "macro.dbt.default__create_table_as": { + "arguments": [], + "created_at": 1689766376.517365, + "depends_on": { + "macros": [ + "macro.dbt.get_assert_columns_equivalent", + "macro.dbt.get_table_columns_and_constraints", + "macro.dbt.get_select_subquery" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", + "meta": {}, + "name": "default__create_table_as", + "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/create_table_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_table_as" + }, + "macro.dbt.default__create_view_as": { + "arguments": [], + "created_at": 1689766376.52372, + "depends_on": { + "macros": [ + "macro.dbt.get_assert_columns_equivalent" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", + "meta": {}, + "name": "default__create_view_as", + "original_file_path": "macros/materializations/models/view/create_view_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/create_view_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_view_as" + }, + "macro.dbt.default__current_timestamp": { + "arguments": [], + "created_at": 1689766376.563894, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__current_timestamp", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__current_timestamp" + }, + "macro.dbt.default__current_timestamp_backcompat": { + "arguments": [], + "created_at": 1689766376.5643861, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", + "meta": {}, + "name": "default__current_timestamp_backcompat", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__current_timestamp_backcompat" + }, + "macro.dbt.default__current_timestamp_in_utc_backcompat": { + "arguments": [], + "created_at": 1689766376.564786, + "depends_on": { + "macros": [ + "macro.dbt.current_timestamp_backcompat", + "macro.dbt.default__current_timestamp_backcompat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__current_timestamp_in_utc_backcompat", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat" + }, + "macro.dbt.default__date_trunc": { + "arguments": [], + "created_at": 1689766376.5613172, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", + "meta": {}, + "name": "default__date_trunc", + "original_file_path": "macros/utils/date_trunc.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date_trunc.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__date_trunc" + }, + "macro.dbt.default__dateadd": { + "arguments": [], + "created_at": 1689766376.550127, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", + "meta": {}, + "name": "default__dateadd", + "original_file_path": "macros/utils/dateadd.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/dateadd.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__dateadd" + }, + "macro.dbt.default__datediff": { + "arguments": [], + "created_at": 1689766376.552931, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", + "meta": {}, + "name": "default__datediff", + "original_file_path": "macros/utils/datediff.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/datediff.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__datediff" + }, + "macro.dbt.default__drop_relation": { + "arguments": [], + "created_at": 1689766376.571219, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n {%- endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__drop_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__drop_relation" + }, + "macro.dbt.default__drop_schema": { + "arguments": [], + "created_at": 1689766376.5632498, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__drop_schema", + "original_file_path": "macros/adapters/schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__drop_schema" + }, + "macro.dbt.default__escape_single_quotes": { + "arguments": [], + "created_at": 1689766376.550852, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", + "meta": {}, + "name": "default__escape_single_quotes", + "original_file_path": "macros/utils/escape_single_quotes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/escape_single_quotes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__escape_single_quotes" + }, + "macro.dbt.default__except": { + "arguments": [], + "created_at": 1689766376.548342, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", + "meta": {}, + "name": "default__except", + "original_file_path": "macros/utils/except.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/except.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__except" + }, + "macro.dbt.default__format_column": { + "arguments": [], + "created_at": 1689766376.5124202, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", + "meta": {}, + "name": "default__format_column", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__format_column" + }, + "macro.dbt.default__generate_alias_name": { + "arguments": [], + "created_at": 1689766376.537911, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__generate_alias_name", + "original_file_path": "macros/get_custom_name/get_custom_alias.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_alias.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__generate_alias_name" + }, + "macro.dbt.default__generate_database_name": { + "arguments": [], + "created_at": 1689766376.539774, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__generate_database_name", + "original_file_path": "macros/get_custom_name/get_custom_database.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_database.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__generate_database_name" + }, + "macro.dbt.default__generate_schema_name": { + "arguments": [], + "created_at": 1689766376.5388072, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__generate_schema_name", + "original_file_path": "macros/get_custom_name/get_custom_schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__generate_schema_name" + }, + "macro.dbt.default__get_assert_columns_equivalent": { + "arguments": [], + "created_at": 1689766376.510565, + "depends_on": { + "macros": [ + "macro.dbt.assert_columns_equivalent" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_assert_columns_equivalent", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_assert_columns_equivalent" + }, + "macro.dbt.default__get_batch_size": { + "arguments": [], + "created_at": 1689766376.535111, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_batch_size", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_batch_size" + }, + "macro.dbt.default__get_binding_char": { + "arguments": [], + "created_at": 1689766376.534821, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_binding_char", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_binding_char" + }, + "macro.dbt.default__get_catalog": { + "arguments": [], + "created_at": 1689766376.585314, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_catalog", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_catalog" + }, + "macro.dbt.default__get_columns_in_query": { + "arguments": [], + "created_at": 1689766376.591773, + "depends_on": { + "macros": [ + "macro.dbt.statement", + "macro.dbt.get_empty_subquery_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_columns_in_query", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_columns_in_query" + }, + "macro.dbt.default__get_columns_in_relation": { + "arguments": [], + "created_at": 1689766376.589469, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_columns_in_relation", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_columns_in_relation" + }, + "macro.dbt.default__get_create_index_sql": { + "arguments": [], + "created_at": 1689766376.5653892, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", + "meta": {}, + "name": "default__get_create_index_sql", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_index_sql" + }, + "macro.dbt.default__get_create_table_as_sql": { + "arguments": [], + "created_at": 1689766376.516255, + "depends_on": { + "macros": [ + "macro.dbt.create_table_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_create_table_as_sql", + "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/create_table_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_table_as_sql" + }, + "macro.dbt.default__get_create_view_as_sql": { + "arguments": [], + "created_at": 1689766376.523107, + "depends_on": { + "macros": [ + "macro.dbt.create_view_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_create_view_as_sql", + "original_file_path": "macros/materializations/models/view/create_view_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/create_view_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_view_as_sql" + }, + "macro.dbt.default__get_csv_sql": { + "arguments": [], + "created_at": 1689766376.534561, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_csv_sql", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_csv_sql" + }, + "macro.dbt.default__get_dcl_statement_list": { + "arguments": [], + "created_at": 1689766376.579158, + "depends_on": { + "macros": [ + "macro.dbt.support_multiple_grantees_per_dcl_statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_dcl_statement_list", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_dcl_statement_list" + }, + "macro.dbt.default__get_delete_insert_merge_sql": { + "arguments": [], + "created_at": 1689766376.488962, + "depends_on": { + "macros": [ + "macro.dbt.get_quoted_csv" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last}}\n {% endfor %}\n {% if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {% endif %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", + "meta": {}, + "name": "default__get_delete_insert_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_delete_insert_merge_sql" + }, + "macro.dbt.default__get_empty_schema_sql": { + "arguments": [], + "created_at": 1689766376.5909078, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {{ col_err.append(col['name']) }}\n {%- endif -%}\n cast(null as {{ col['data_type'] }}) as {{ col['name'] }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- endif -%}\n{% endmacro %}", + "meta": {}, + "name": "default__get_empty_schema_sql", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_empty_schema_sql" + }, + "macro.dbt.default__get_empty_subquery_sql": { + "arguments": [], + "created_at": 1689766376.59008, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql) %}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", + "meta": {}, + "name": "default__get_empty_subquery_sql", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_empty_subquery_sql" + }, + "macro.dbt.default__get_grant_sql": { + "arguments": [], + "created_at": 1689766376.577594, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "default__get_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_grant_sql" + }, + "macro.dbt.default__get_incremental_append_sql": { + "arguments": [], + "created_at": 1689766376.4917629, + "depends_on": { + "macros": [ + "macro.dbt.get_insert_into_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_append_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_append_sql" + }, + "macro.dbt.default__get_incremental_default_sql": { + "arguments": [], + "created_at": 1689766376.4935958, + "depends_on": { + "macros": [ + "macro.dbt.get_incremental_append_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_default_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_default_sql" + }, + "macro.dbt.default__get_incremental_delete_insert_sql": { + "arguments": [], + "created_at": 1689766376.4922712, + "depends_on": { + "macros": [ + "macro.dbt.get_delete_insert_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_delete_insert_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql" + }, + "macro.dbt.default__get_incremental_insert_overwrite_sql": { + "arguments": [], + "created_at": 1689766376.49324, + "depends_on": { + "macros": [ + "macro.dbt.get_insert_overwrite_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_insert_overwrite_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql" + }, + "macro.dbt.default__get_incremental_merge_sql": { + "arguments": [], + "created_at": 1689766376.492772, + "depends_on": { + "macros": [ + "macro.dbt.get_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_merge_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_merge_sql" + }, + "macro.dbt.default__get_insert_overwrite_merge_sql": { + "arguments": [], + "created_at": 1689766376.489915, + "depends_on": { + "macros": [ + "macro.dbt.get_quoted_csv" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_insert_overwrite_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql" + }, + "macro.dbt.default__get_merge_sql": { + "arguments": [], + "created_at": 1689766376.487649, + "depends_on": { + "macros": [ + "macro.dbt.get_quoted_csv", + "macro.dbt.get_merge_update_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_merge_sql" + }, + "macro.dbt.default__get_merge_update_columns": { + "arguments": [], + "created_at": 1689766376.478741, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_merge_update_columns", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_merge_update_columns" + }, + "macro.dbt.default__get_or_create_relation": { + "arguments": [], + "created_at": 1689766376.572939, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", + "meta": {}, + "name": "default__get_or_create_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_or_create_relation" + }, + "macro.dbt.default__get_revoke_sql": { + "arguments": [], + "created_at": 1689766376.578012, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "default__get_revoke_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_revoke_sql" + }, + "macro.dbt.default__get_select_subquery": { + "arguments": [], + "created_at": 1689766376.517819, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select\n {% for column in model['columns'] %}\n {{ column }}{{ \", \" if not loop.last }}\n {% endfor %}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", + "meta": {}, + "name": "default__get_select_subquery", + "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/create_table_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_select_subquery" + }, + "macro.dbt.default__get_show_grant_sql": { + "arguments": [], + "created_at": 1689766376.577173, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_show_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_show_grant_sql" + }, + "macro.dbt.default__get_table_columns_and_constraints": { + "arguments": [], + "created_at": 1689766376.5097039, + "depends_on": { + "macros": [ + "macro.dbt.table_columns_and_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_table_columns_and_constraints", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_table_columns_and_constraints" + }, + "macro.dbt.default__get_test_sql": { + "arguments": [], + "created_at": 1689766376.474005, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", + "meta": {}, + "name": "default__get_test_sql", + "original_file_path": "macros/materializations/tests/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_test_sql" + }, + "macro.dbt.default__get_true_sql": { + "arguments": [], + "created_at": 1689766376.4621289, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_true_sql", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_true_sql" + }, + "macro.dbt.default__get_where_subquery": { + "arguments": [], + "created_at": 1689766376.474785, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_where_subquery", + "original_file_path": "macros/materializations/tests/where_subquery.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/where_subquery.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_where_subquery" + }, + "macro.dbt.default__handle_existing_table": { + "arguments": [], + "created_at": 1689766376.520983, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "default__handle_existing_table", + "original_file_path": "macros/materializations/models/view/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__handle_existing_table" + }, + "macro.dbt.default__hash": { + "arguments": [], + "created_at": 1689766376.553776, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", + "meta": {}, + "name": "default__hash", + "original_file_path": "macros/utils/hash.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/hash.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__hash" + }, + "macro.dbt.default__information_schema_name": { + "arguments": [], + "created_at": 1689766376.5856798, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", + "meta": {}, + "name": "default__information_schema_name", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__information_schema_name" + }, + "macro.dbt.default__intersect": { + "arguments": [], + "created_at": 1689766376.550436, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", + "meta": {}, + "name": "default__intersect", + "original_file_path": "macros/utils/intersect.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/intersect.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__intersect" + }, + "macro.dbt.default__last_day": { + "arguments": [], + "created_at": 1689766376.5599139, + "depends_on": { + "macros": [ + "macro.dbt.default_last_day" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", + "meta": {}, + "name": "default__last_day", + "original_file_path": "macros/utils/last_day.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/last_day.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__last_day" + }, + "macro.dbt.default__length": { + "arguments": [], + "created_at": 1689766376.549609, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__length", + "original_file_path": "macros/utils/length.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/length.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__length" + }, + "macro.dbt.default__list_relations_without_caching": { + "arguments": [], + "created_at": 1689766376.587014, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__list_relations_without_caching", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__list_relations_without_caching" + }, + "macro.dbt.default__list_schemas": { + "arguments": [], + "created_at": 1689766376.586134, + "depends_on": { + "macros": [ + "macro.dbt.information_schema_name", + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__list_schemas", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__list_schemas" + }, + "macro.dbt.default__listagg": { + "arguments": [], + "created_at": 1689766376.552421, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__listagg", + "original_file_path": "macros/utils/listagg.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/listagg.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__listagg" + }, + "macro.dbt.default__load_csv_rows": { + "arguments": [], + "created_at": 1689766376.53708, + "depends_on": { + "macros": [ + "macro.dbt.get_batch_size", + "macro.dbt.get_seed_column_quoted_csv", + "macro.dbt.get_binding_char" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", + "meta": {}, + "name": "default__load_csv_rows", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__load_csv_rows" + }, + "macro.dbt.default__make_backup_relation": { + "arguments": [], + "created_at": 1689766376.570791, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "default__make_backup_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__make_backup_relation" + }, + "macro.dbt.default__make_intermediate_relation": { + "arguments": [], + "created_at": 1689766376.569613, + "depends_on": { + "macros": [ + "macro.dbt.default__make_temp_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__make_intermediate_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__make_intermediate_relation" + }, + "macro.dbt.default__make_temp_relation": { + "arguments": [], + "created_at": 1689766376.5701299, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "default__make_temp_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__make_temp_relation" + }, + "macro.dbt.default__persist_docs": { + "arguments": [], + "created_at": 1689766376.583472, + "depends_on": { + "macros": [ + "macro.dbt.run_query", + "macro.dbt.alter_relation_comment", + "macro.dbt.alter_column_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "default__persist_docs", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__persist_docs" + }, + "macro.dbt.default__position": { + "arguments": [], + "created_at": 1689766376.554983, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__position", + "original_file_path": "macros/utils/position.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/position.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__position" + }, + "macro.dbt.default__post_snapshot": { + "arguments": [], + "created_at": 1689766376.461851, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", + "meta": {}, + "name": "default__post_snapshot", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__post_snapshot" + }, + "macro.dbt.default__rename_relation": { + "arguments": [], + "created_at": 1689766376.572056, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__rename_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__rename_relation" + }, + "macro.dbt.default__replace": { + "arguments": [], + "created_at": 1689766376.548854, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", + "meta": {}, + "name": "default__replace", + "original_file_path": "macros/utils/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__replace" + }, + "macro.dbt.default__reset_csv_table": { + "arguments": [], + "created_at": 1689766376.534233, + "depends_on": { + "macros": [ + "macro.dbt.create_csv_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", + "meta": {}, + "name": "default__reset_csv_table", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__reset_csv_table" + }, + "macro.dbt.default__resolve_model_name": { + "arguments": [], + "created_at": 1689766376.59554, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "default__resolve_model_name", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__resolve_model_name" + }, + "macro.dbt.default__right": { + "arguments": [], + "created_at": 1689766376.5513558, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__right", + "original_file_path": "macros/utils/right.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/right.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__right" + }, + "macro.dbt.default__safe_cast": { + "arguments": [], + "created_at": 1689766376.553361, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", + "meta": {}, + "name": "default__safe_cast", + "original_file_path": "macros/utils/safe_cast.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/safe_cast.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__safe_cast" + }, + "macro.dbt.default__snapshot_get_time": { + "arguments": [], + "created_at": 1689766376.564141, + "depends_on": { + "macros": [ + "macro.dbt.current_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", + "meta": {}, + "name": "default__snapshot_get_time", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_get_time" + }, + "macro.dbt.default__snapshot_hash_arguments": { + "arguments": [], + "created_at": 1689766376.452668, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", + "meta": {}, + "name": "default__snapshot_hash_arguments", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_hash_arguments" + }, + "macro.dbt.default__snapshot_merge_sql": { + "arguments": [], + "created_at": 1689766376.4484491, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", + "meta": {}, + "name": "default__snapshot_merge_sql", + "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/snapshot_merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_merge_sql" + }, + "macro.dbt.default__snapshot_staging_table": { + "arguments": [], + "created_at": 1689766376.463292, + "depends_on": { + "macros": [ + "macro.dbt.snapshot_get_time" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__snapshot_staging_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_staging_table" + }, + "macro.dbt.default__snapshot_string_as_time": { + "arguments": [], + "created_at": 1689766376.4537668, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", + "meta": {}, + "name": "default__snapshot_string_as_time", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_string_as_time" + }, + "macro.dbt.default__split_part": { + "arguments": [], + "created_at": 1689766376.560687, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", + "meta": {}, + "name": "default__split_part", + "original_file_path": "macros/utils/split_part.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/split_part.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__split_part" + }, + "macro.dbt.default__string_literal": { + "arguments": [], + "created_at": 1689766376.5553322, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", + "meta": {}, + "name": "default__string_literal", + "original_file_path": "macros/utils/literal.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/literal.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__string_literal" + }, + "macro.dbt.default__support_multiple_grantees_per_dcl_statement": { + "arguments": [], + "created_at": 1689766376.576531, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "default__support_multiple_grantees_per_dcl_statement", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement" + }, + "macro.dbt.default__test_accepted_values": { + "arguments": [], + "created_at": 1689766376.54125, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", + "meta": {}, + "name": "default__test_accepted_values", + "original_file_path": "macros/generic_test_sql/accepted_values.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/generic_test_sql/accepted_values.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__test_accepted_values" + }, + "macro.dbt.default__test_not_null": { + "arguments": [], + "created_at": 1689766376.5404239, + "depends_on": { + "macros": [ + "macro.dbt.should_store_failures" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", + "meta": {}, + "name": "default__test_not_null", + "original_file_path": "macros/generic_test_sql/not_null.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/generic_test_sql/not_null.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__test_not_null" + }, + "macro.dbt.default__test_relationships": { + "arguments": [], + "created_at": 1689766376.5401258, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", + "meta": {}, + "name": "default__test_relationships", + "original_file_path": "macros/generic_test_sql/relationships.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/generic_test_sql/relationships.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__test_relationships" + }, + "macro.dbt.default__test_unique": { + "arguments": [], + "created_at": 1689766376.540672, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", + "meta": {}, + "name": "default__test_unique", + "original_file_path": "macros/generic_test_sql/unique.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/generic_test_sql/unique.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__test_unique" + }, + "macro.dbt.default__truncate_relation": { + "arguments": [], + "created_at": 1689766376.571568, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__truncate_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__truncate_relation" + }, + "macro.dbt.default__type_bigint": { + "arguments": [], + "created_at": 1689766376.55777, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_bigint", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_bigint" + }, + "macro.dbt.default__type_boolean": { + "arguments": [], + "created_at": 1689766376.558392, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "default__type_boolean", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_boolean" + }, + "macro.dbt.default__type_float": { + "arguments": [], + "created_at": 1689766376.557118, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_float", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_float" + }, + "macro.dbt.default__type_int": { + "arguments": [], + "created_at": 1689766376.558081, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "default__type_int", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_int" + }, + "macro.dbt.default__type_numeric": { + "arguments": [], + "created_at": 1689766376.557457, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_numeric", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_numeric" + }, + "macro.dbt.default__type_string": { + "arguments": [], + "created_at": 1689766376.556418, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_string", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_string" + }, + "macro.dbt.default__type_timestamp": { + "arguments": [], + "created_at": 1689766376.556797, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_timestamp", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_timestamp" + }, + "macro.dbt.default_last_day": { + "arguments": [], + "created_at": 1689766376.559766, + "depends_on": { + "macros": [ + "macro.dbt.dateadd", + "macro.dbt.date_trunc" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "default_last_day", + "original_file_path": "macros/utils/last_day.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/last_day.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default_last_day" + }, + "macro.dbt.diff_column_data_types": { + "arguments": [], + "created_at": 1689766376.4777448, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", + "meta": {}, + "name": "diff_column_data_types", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.diff_column_data_types" + }, + "macro.dbt.diff_columns": { + "arguments": [], + "created_at": 1689766376.477046, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", + "meta": {}, + "name": "diff_columns", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.diff_columns" + }, + "macro.dbt.drop_relation": { + "arguments": [], + "created_at": 1689766376.570985, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__drop_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "drop_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.drop_relation" + }, + "macro.dbt.drop_relation_if_exists": { + "arguments": [], + "created_at": 1689766376.573519, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "drop_relation_if_exists", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.drop_relation_if_exists" + }, + "macro.dbt.drop_schema": { + "arguments": [], + "created_at": 1689766376.56307, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__drop_schema" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", + "meta": {}, + "name": "drop_schema", + "original_file_path": "macros/adapters/schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.drop_schema" + }, + "macro.dbt.escape_single_quotes": { + "arguments": [], + "created_at": 1689766376.5507042, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__escape_single_quotes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "escape_single_quotes", + "original_file_path": "macros/utils/escape_single_quotes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/escape_single_quotes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.escape_single_quotes" + }, + "macro.dbt.except": { + "arguments": [], + "created_at": 1689766376.5482602, + "depends_on": { + "macros": [ + "macro.dbt.default__except" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "except", + "original_file_path": "macros/utils/except.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/except.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.except" + }, + "macro.dbt.format_columns": { + "arguments": [], + "created_at": 1689766376.5120618, + "depends_on": { + "macros": [ + "macro.dbt.default__format_column" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", + "meta": {}, + "name": "format_columns", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.format_columns" + }, + "macro.dbt.generate_alias_name": { + "arguments": [], + "created_at": 1689766376.537523, + "depends_on": { + "macros": [ + "macro.dbt.default__generate_alias_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", + "meta": {}, + "name": "generate_alias_name", + "original_file_path": "macros/get_custom_name/get_custom_alias.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_alias.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.generate_alias_name" + }, + "macro.dbt.generate_database_name": { + "arguments": [], + "created_at": 1689766376.539517, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__generate_database_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", + "meta": {}, + "name": "generate_database_name", + "original_file_path": "macros/get_custom_name/get_custom_database.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_database.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.generate_database_name" + }, + "macro.dbt.generate_schema_name": { + "arguments": [], + "created_at": 1689766376.538522, + "depends_on": { + "macros": [ + "macro.dbt.default__generate_schema_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", + "meta": {}, + "name": "generate_schema_name", + "original_file_path": "macros/get_custom_name/get_custom_schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.generate_schema_name" + }, + "macro.dbt.generate_schema_name_for_env": { + "arguments": [], + "created_at": 1689766376.539108, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "generate_schema_name_for_env", + "original_file_path": "macros/get_custom_name/get_custom_schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.generate_schema_name_for_env" + }, + "macro.dbt.get_assert_columns_equivalent": { + "arguments": [], + "created_at": 1689766376.510425, + "depends_on": { + "macros": [ + "macro.dbt.default__get_assert_columns_equivalent" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "get_assert_columns_equivalent", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_assert_columns_equivalent" + }, + "macro.dbt.get_batch_size": { + "arguments": [], + "created_at": 1689766376.534988, + "depends_on": { + "macros": [ + "macro.dbt.default__get_batch_size" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_batch_size", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_batch_size" + }, + "macro.dbt.get_binding_char": { + "arguments": [], + "created_at": 1689766376.534705, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__get_binding_char" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", + "meta": {}, + "name": "get_binding_char", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_binding_char" + }, + "macro.dbt.get_catalog": { + "arguments": [], + "created_at": 1689766376.585037, + "depends_on": { + "macros": [ + "macro.dbt.default__get_catalog" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_catalog", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_catalog" + }, + "macro.dbt.get_column_schema_from_query": { + "arguments": [], + "created_at": 1689766376.591232, + "depends_on": { + "macros": [ + "macro.dbt.get_empty_subquery_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_column_schema_from_query(select_sql) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", + "meta": {}, + "name": "get_column_schema_from_query", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_column_schema_from_query" + }, + "macro.dbt.get_columns_in_query": { + "arguments": [], + "created_at": 1689766376.591417, + "depends_on": { + "macros": [ + "macro.dbt.default__get_columns_in_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_columns_in_query", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_columns_in_query" + }, + "macro.dbt.get_columns_in_relation": { + "arguments": [], + "created_at": 1689766376.589223, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__get_columns_in_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_columns_in_relation", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_columns_in_relation" + }, + "macro.dbt.get_create_index_sql": { + "arguments": [], + "created_at": 1689766376.5652502, + "depends_on": { + "macros": [ + "macro.dbt.default__get_create_index_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_create_index_sql", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_create_index_sql" + }, + "macro.dbt.get_create_table_as_sql": { + "arguments": [], + "created_at": 1689766376.516062, + "depends_on": { + "macros": [ + "macro.dbt.default__get_create_table_as_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_create_table_as_sql", + "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/create_table_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_create_table_as_sql" + }, + "macro.dbt.get_create_view_as_sql": { + "arguments": [], + "created_at": 1689766376.522943, + "depends_on": { + "macros": [ + "macro.dbt.default__get_create_view_as_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_create_view_as_sql", + "original_file_path": "macros/materializations/models/view/create_view_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/create_view_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_create_view_as_sql" + }, + "macro.dbt.get_csv_sql": { + "arguments": [], + "created_at": 1689766376.534427, + "depends_on": { + "macros": [ + "macro.dbt.default__get_csv_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", + "meta": {}, + "name": "get_csv_sql", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_csv_sql" + }, + "macro.dbt.get_dcl_statement_list": { + "arguments": [], + "created_at": 1689766376.578248, + "depends_on": { + "macros": [ + "macro.dbt.default__get_dcl_statement_list" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_dcl_statement_list", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_dcl_statement_list" + }, + "macro.dbt.get_delete_insert_merge_sql": { + "arguments": [], + "created_at": 1689766376.487926, + "depends_on": { + "macros": [ + "macro.dbt.default__get_delete_insert_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_delete_insert_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_delete_insert_merge_sql" + }, + "macro.dbt.get_empty_schema_sql": { + "arguments": [], + "created_at": 1689766376.59026, + "depends_on": { + "macros": [ + "macro.dbt.default__get_empty_schema_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_empty_schema_sql", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_empty_schema_sql" + }, + "macro.dbt.get_empty_subquery_sql": { + "arguments": [], + "created_at": 1689766376.589969, + "depends_on": { + "macros": [ + "macro.dbt.default__get_empty_subquery_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_empty_subquery_sql(select_sql) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_empty_subquery_sql", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_empty_subquery_sql" + }, + "macro.dbt.get_grant_sql": { + "arguments": [], + "created_at": 1689766376.577402, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__get_grant_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_grant_sql" + }, + "macro.dbt.get_incremental_append_sql": { + "arguments": [], + "created_at": 1689766376.49151, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__get_incremental_append_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_append_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_append_sql" + }, + "macro.dbt.get_incremental_default_sql": { + "arguments": [], + "created_at": 1689766376.493441, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__get_incremental_default_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_default_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_default_sql" + }, + "macro.dbt.get_incremental_delete_insert_sql": { + "arguments": [], + "created_at": 1689766376.491967, + "depends_on": { + "macros": [ + "macro.dbt.default__get_incremental_delete_insert_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_delete_insert_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_delete_insert_sql" + }, + "macro.dbt.get_incremental_insert_overwrite_sql": { + "arguments": [], + "created_at": 1689766376.492967, + "depends_on": { + "macros": [ + "macro.dbt.default__get_incremental_insert_overwrite_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_insert_overwrite_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql" + }, + "macro.dbt.get_incremental_merge_sql": { + "arguments": [], + "created_at": 1689766376.492467, + "depends_on": { + "macros": [ + "macro.dbt.default__get_incremental_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_merge_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_merge_sql" + }, + "macro.dbt.get_insert_into_sql": { + "arguments": [], + "created_at": 1689766376.493896, + "depends_on": { + "macros": [ + "macro.dbt.get_quoted_csv" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", + "meta": {}, + "name": "get_insert_into_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_insert_into_sql" + }, + "macro.dbt.get_insert_overwrite_merge_sql": { + "arguments": [], + "created_at": 1689766376.4892402, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_insert_overwrite_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_insert_overwrite_merge_sql" + }, + "macro.dbt.get_merge_sql": { + "arguments": [], + "created_at": 1689766376.485783, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__get_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_merge_sql" + }, + "macro.dbt.get_merge_update_columns": { + "arguments": [], + "created_at": 1689766376.4779892, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__get_merge_update_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_merge_update_columns", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_merge_update_columns" + }, + "macro.dbt.get_or_create_relation": { + "arguments": [], + "created_at": 1689766376.572375, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__get_or_create_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_or_create_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_or_create_relation" + }, + "macro.dbt.get_quoted_csv": { + "arguments": [], + "created_at": 1689766376.476466, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_quoted_csv", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_quoted_csv" + }, + "macro.dbt.get_revoke_sql": { + "arguments": [], + "created_at": 1689766376.577824, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__get_revoke_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_revoke_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_revoke_sql" + }, + "macro.dbt.get_seed_column_quoted_csv": { + "arguments": [], + "created_at": 1689766376.535594, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", + "meta": {}, + "name": "get_seed_column_quoted_csv", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_seed_column_quoted_csv" + }, + "macro.dbt.get_select_subquery": { + "arguments": [], + "created_at": 1689766376.517558, + "depends_on": { + "macros": [ + "macro.dbt.default__get_select_subquery" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_select_subquery", + "original_file_path": "macros/materializations/models/table/create_table_as.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/create_table_as.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_select_subquery" + }, + "macro.dbt.get_show_grant_sql": { + "arguments": [], + "created_at": 1689766376.577069, + "depends_on": { + "macros": [ + "macro.dbt.default__get_show_grant_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_show_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_show_grant_sql" + }, + "macro.dbt.get_table_columns_and_constraints": { + "arguments": [], + "created_at": 1689766376.509569, + "depends_on": { + "macros": [ + "macro.dbt.default__get_table_columns_and_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "get_table_columns_and_constraints", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_table_columns_and_constraints" + }, + "macro.dbt.get_test_sql": { + "arguments": [], + "created_at": 1689766376.473691, + "depends_on": { + "macros": [ + "macro.dbt.default__get_test_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_test_sql", + "original_file_path": "macros/materializations/tests/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_test_sql" + }, + "macro.dbt.get_true_sql": { + "arguments": [], + "created_at": 1689766376.462003, + "depends_on": { + "macros": [ + "macro.dbt.default__get_true_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", + "meta": {}, + "name": "get_true_sql", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_true_sql" + }, + "macro.dbt.get_where_subquery": { + "arguments": [], + "created_at": 1689766376.474396, + "depends_on": { + "macros": [ + "macro.dbt.default__get_where_subquery" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", + "meta": {}, + "name": "get_where_subquery", + "original_file_path": "macros/materializations/tests/where_subquery.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/where_subquery.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_where_subquery" + }, + "macro.dbt.handle_existing_table": { + "arguments": [], + "created_at": 1689766376.5207539, + "depends_on": { + "macros": [ + "macro.dbt.default__handle_existing_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "handle_existing_table", + "original_file_path": "macros/materializations/models/view/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.handle_existing_table" + }, + "macro.dbt.hash": { + "arguments": [], + "created_at": 1689766376.553617, + "depends_on": { + "macros": [ + "macro.dbt.default__hash" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", + "meta": {}, + "name": "hash", + "original_file_path": "macros/utils/hash.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/hash.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.hash" + }, + "macro.dbt.in_transaction": { + "arguments": [], + "created_at": 1689766376.4462988, + "depends_on": { + "macros": [ + "macro.dbt.make_hook_config" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", + "meta": {}, + "name": "in_transaction", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.in_transaction" + }, + "macro.dbt.incremental_validate_on_schema_change": { + "arguments": [], + "created_at": 1689766376.505162, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "incremental_validate_on_schema_change", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.incremental_validate_on_schema_change" + }, + "macro.dbt.information_schema_name": { + "arguments": [], + "created_at": 1689766376.585514, + "depends_on": { + "macros": [ + "macro.dbt.default__information_schema_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", + "meta": {}, + "name": "information_schema_name", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.information_schema_name" + }, + "macro.dbt.intersect": { + "arguments": [], + "created_at": 1689766376.550354, + "depends_on": { + "macros": [ + "macro.dbt.default__intersect" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "intersect", + "original_file_path": "macros/utils/intersect.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/intersect.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.intersect" + }, + "macro.dbt.is_incremental": { + "arguments": [], + "created_at": 1689766376.490581, + "depends_on": { + "macros": [ + "macro.dbt.should_full_refresh" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "is_incremental", + "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/is_incremental.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.is_incremental" + }, + "macro.dbt.last_day": { + "arguments": [], + "created_at": 1689766376.559493, + "depends_on": { + "macros": [ + "macro.dbt.default__last_day" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", + "meta": {}, + "name": "last_day", + "original_file_path": "macros/utils/last_day.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/last_day.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.last_day" + }, + "macro.dbt.length": { + "arguments": [], + "created_at": 1689766376.5494988, + "depends_on": { + "macros": [ + "macro.dbt.default__length" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "length", + "original_file_path": "macros/utils/length.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/length.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.length" + }, + "macro.dbt.list_relations_without_caching": { + "arguments": [], + "created_at": 1689766376.5868459, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__list_relations_without_caching" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "list_relations_without_caching", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.list_relations_without_caching" + }, + "macro.dbt.list_schemas": { + "arguments": [], + "created_at": 1689766376.58587, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__list_schemas" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", + "meta": {}, + "name": "list_schemas", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.list_schemas" + }, + "macro.dbt.listagg": { + "arguments": [], + "created_at": 1689766376.55202, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__listagg" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", + "meta": {}, + "name": "listagg", + "original_file_path": "macros/utils/listagg.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/listagg.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.listagg" + }, + "macro.dbt.load_cached_relation": { + "arguments": [], + "created_at": 1689766376.573177, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", + "meta": {}, + "name": "load_cached_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.load_cached_relation" + }, + "macro.dbt.load_csv_rows": { + "arguments": [], + "created_at": 1689766376.5357802, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__load_csv_rows" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", + "meta": {}, + "name": "load_csv_rows", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.load_csv_rows" + }, + "macro.dbt.load_relation": { + "arguments": [], + "created_at": 1689766376.5733209, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "load_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.load_relation" + }, + "macro.dbt.make_backup_relation": { + "arguments": [], + "created_at": 1689766376.570415, + "depends_on": { + "macros": [ + "macro.dbt.default__make_backup_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "make_backup_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_backup_relation" + }, + "macro.dbt.make_hook_config": { + "arguments": [], + "created_at": 1689766376.446005, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", + "meta": {}, + "name": "make_hook_config", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_hook_config" + }, + "macro.dbt.make_intermediate_relation": { + "arguments": [], + "created_at": 1689766376.5694451, + "depends_on": { + "macros": [ + "macro.dbt.default__make_intermediate_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "make_intermediate_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_intermediate_relation" + }, + "macro.dbt.make_temp_relation": { + "arguments": [], + "created_at": 1689766376.569835, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__make_temp_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "make_temp_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_temp_relation" + }, + "macro.dbt.materialization_incremental_default": { + "arguments": [], + "created_at": 1689766376.499119, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.make_temp_relation", + "macro.dbt.make_intermediate_relation", + "macro.dbt.make_backup_relation", + "macro.dbt.should_full_refresh", + "macro.dbt.incremental_validate_on_schema_change", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.run_hooks", + "macro.dbt.get_create_table_as_sql", + "macro.dbt.run_query", + "macro.dbt.process_schema_changes", + "macro.dbt.statement", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt.create_indexes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparison later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", + "meta": {}, + "name": "materialization_incremental_default", + "original_file_path": "macros/materializations/models/incremental/incremental.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/incremental.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_incremental_default" + }, + "macro.dbt.materialization_seed_default": { + "arguments": [], + "created_at": 1689766376.527106, + "depends_on": { + "macros": [ + "macro.dbt.should_full_refresh", + "macro.dbt.run_hooks", + "macro.dbt.reset_csv_table", + "macro.dbt.create_csv_table", + "macro.dbt.load_csv_rows", + "macro.dbt.noop_statement", + "macro.dbt.get_csv_sql", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt.create_indexes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_seed_default", + "original_file_path": "macros/materializations/seeds/seed.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/seed.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_seed_default" + }, + "macro.dbt.materialization_snapshot_default": { + "arguments": [], + "created_at": 1689766376.470805, + "depends_on": { + "macros": [ + "macro.dbt.get_or_create_relation", + "macro.dbt.run_hooks", + "macro.dbt.strategy_dispatch", + "macro.dbt.build_snapshot_table", + "macro.dbt.create_table_as", + "macro.dbt.build_snapshot_staging_table", + "macro.dbt.create_columns", + "macro.dbt.snapshot_merge_sql", + "macro.dbt.statement", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt.create_indexes", + "macro.dbt.post_snapshot" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparison later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_snapshot_default", + "original_file_path": "macros/materializations/snapshots/snapshot.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/snapshot.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_snapshot_default" + }, + "macro.dbt.materialization_table_default": { + "arguments": [], + "created_at": 1689766376.51514, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.make_intermediate_relation", + "macro.dbt.make_backup_relation", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.run_hooks", + "macro.dbt.statement", + "macro.dbt.get_create_table_as_sql", + "macro.dbt.create_indexes", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparison later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_table_default", + "original_file_path": "macros/materializations/models/table/table.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/table.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_table_default" + }, + "macro.dbt.materialization_test_default": { + "arguments": [], + "created_at": 1689766376.473208, + "depends_on": { + "macros": [ + "macro.dbt.should_store_failures", + "macro.dbt.statement", + "macro.dbt.create_table_as", + "macro.dbt.get_test_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type='table') -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ create_table_as(False, target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", + "meta": {}, + "name": "materialization_test_default", + "original_file_path": "macros/materializations/tests/test.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/test.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_test_default" + }, + "macro.dbt.materialization_view_default": { + "arguments": [], + "created_at": 1689766376.520448, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.make_intermediate_relation", + "macro.dbt.make_backup_relation", + "macro.dbt.run_hooks", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.statement", + "macro.dbt.get_create_view_as_sql", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparison later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", + "meta": {}, + "name": "materialization_view_default", + "original_file_path": "macros/materializations/models/view/view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view/view.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_view_default" + }, + "macro.dbt.noop_statement": { + "arguments": [], + "created_at": 1689766376.543406, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "noop_statement", + "original_file_path": "macros/etc/statement.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/statement.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.noop_statement" + }, + "macro.dbt.partition_range": { + "arguments": [], + "created_at": 1689766376.547775, + "depends_on": { + "macros": [ + "macro.dbt.dates_in_range" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", + "meta": {}, + "name": "partition_range", + "original_file_path": "macros/etc/datetime.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/datetime.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.partition_range" + }, + "macro.dbt.persist_docs": { + "arguments": [], + "created_at": 1689766376.582967, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", + "meta": {}, + "name": "persist_docs", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.persist_docs" + }, + "macro.dbt.position": { + "arguments": [], + "created_at": 1689766376.554846, + "depends_on": { + "macros": [ + "macro.dbt.default__position" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", + "meta": {}, + "name": "position", + "original_file_path": "macros/utils/position.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/position.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.position" + }, + "macro.dbt.post_snapshot": { + "arguments": [], + "created_at": 1689766376.4617548, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__post_snapshot" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "post_snapshot", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.post_snapshot" + }, + "macro.dbt.process_schema_changes": { + "arguments": [], + "created_at": 1689766376.508546, + "depends_on": { + "macros": [ + "macro.dbt.check_for_schema_changes", + "macro.dbt.sync_column_schemas" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "process_schema_changes", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.process_schema_changes" + }, + "macro.dbt.py_current_timestring": { + "arguments": [], + "created_at": 1689766376.548018, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", + "meta": {}, + "name": "py_current_timestring", + "original_file_path": "macros/etc/datetime.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/datetime.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.py_current_timestring" + }, + "macro.dbt.py_script_comment": { + "arguments": [], + "created_at": 1689766376.598182, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", + "meta": {}, + "name": "py_script_comment", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.py_script_comment" + }, + "macro.dbt.py_script_postfix": { + "arguments": [], + "created_at": 1689766376.5980978, + "depends_on": { + "macros": [ + "macro.dbt.build_ref_function", + "macro.dbt.build_source_function", + "macro.dbt.build_config_dict", + "macro.dbt.resolve_model_name", + "macro.dbt.is_incremental", + "macro.dbt.py_script_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", + "meta": {}, + "name": "py_script_postfix", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.py_script_postfix" + }, + "macro.dbt.rename_relation": { + "arguments": [], + "created_at": 1689766376.5717702, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__rename_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "rename_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.rename_relation" + }, + "macro.dbt.replace": { + "arguments": [], + "created_at": 1689766376.5486798, + "depends_on": { + "macros": [ + "macro.dbt.default__replace" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", + "meta": {}, + "name": "replace", + "original_file_path": "macros/utils/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.replace" + }, + "macro.dbt.reset_csv_table": { + "arguments": [], + "created_at": 1689766376.5337431, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__reset_csv_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", + "meta": {}, + "name": "reset_csv_table", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.reset_csv_table" + }, + "macro.dbt.resolve_model_name": { + "arguments": [], + "created_at": 1689766376.5953841, + "depends_on": { + "macros": [ + "macro.dbt.default__resolve_model_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", + "meta": {}, + "name": "resolve_model_name", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.resolve_model_name" + }, + "macro.dbt.right": { + "arguments": [], + "created_at": 1689766376.5511491, + "depends_on": { + "macros": [ + "macro.dbt.default__right" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "right", + "original_file_path": "macros/utils/right.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/right.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.right" + }, + "macro.dbt.run_hooks": { + "arguments": [], + "created_at": 1689766376.4457998, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "run_hooks", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.run_hooks" + }, + "macro.dbt.run_query": { + "arguments": [], + "created_at": 1689766376.543704, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", + "meta": {}, + "name": "run_query", + "original_file_path": "macros/etc/statement.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/statement.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.run_query" + }, + "macro.dbt.safe_cast": { + "arguments": [], + "created_at": 1689766376.553222, + "depends_on": { + "macros": [ + "macro.dbt.default__safe_cast" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", + "meta": {}, + "name": "safe_cast", + "original_file_path": "macros/utils/safe_cast.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/safe_cast.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.safe_cast" + }, + "macro.dbt.set_sql_header": { + "arguments": [], + "created_at": 1689766376.4468238, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", + "meta": {}, + "name": "set_sql_header", + "original_file_path": "macros/materializations/configs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/configs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.set_sql_header" + }, + "macro.dbt.should_full_refresh": { + "arguments": [], + "created_at": 1689766376.447285, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", + "meta": {}, + "name": "should_full_refresh", + "original_file_path": "macros/materializations/configs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/configs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.should_full_refresh" + }, + "macro.dbt.should_revoke": { + "arguments": [], + "created_at": 1689766376.576878, + "depends_on": { + "macros": [ + "macro.dbt.copy_grants" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "should_revoke", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.should_revoke" + }, + "macro.dbt.should_store_failures": { + "arguments": [], + "created_at": 1689766376.447673, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", + "meta": {}, + "name": "should_store_failures", + "original_file_path": "macros/materializations/configs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/configs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.should_store_failures" + }, + "macro.dbt.snapshot_check_all_get_existing_columns": { + "arguments": [], + "created_at": 1689766376.4553568, + "depends_on": { + "macros": [ + "macro.dbt.get_columns_in_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_check_all_get_existing_columns", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns" + }, + "macro.dbt.snapshot_check_strategy": { + "arguments": [], + "created_at": 1689766376.456799, + "depends_on": { + "macros": [ + "macro.dbt.snapshot_get_time", + "macro.dbt.snapshot_check_all_get_existing_columns", + "macro.dbt.get_true_sql", + "macro.dbt.snapshot_hash_arguments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n {% set updated_at = config.get('updated_at', snapshot_get_time()) %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", + "meta": {}, + "name": "snapshot_check_strategy", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_check_strategy" + }, + "macro.dbt.snapshot_get_time": { + "arguments": [], + "created_at": 1689766376.5640352, + "depends_on": { + "macros": [ + "macro.dbt.default__snapshot_get_time" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "snapshot_get_time", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_get_time" + }, + "macro.dbt.snapshot_hash_arguments": { + "arguments": [], + "created_at": 1689766376.4524229, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__snapshot_hash_arguments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_hash_arguments", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_hash_arguments" + }, + "macro.dbt.snapshot_merge_sql": { + "arguments": [], + "created_at": 1689766376.448155, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__snapshot_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_merge_sql", + "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/snapshot_merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_merge_sql" + }, + "macro.dbt.snapshot_staging_table": { + "arguments": [], + "created_at": 1689766376.462346, + "depends_on": { + "macros": [ + "macro.dbt.default__snapshot_staging_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "snapshot_staging_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_staging_table" + }, + "macro.dbt.snapshot_string_as_time": { + "arguments": [], + "created_at": 1689766376.453588, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__snapshot_string_as_time" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_string_as_time", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_string_as_time" + }, + "macro.dbt.snapshot_timestamp_strategy": { + "arguments": [], + "created_at": 1689766376.4534159, + "depends_on": { + "macros": [ + "macro.dbt.snapshot_hash_arguments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", + "meta": {}, + "name": "snapshot_timestamp_strategy", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_timestamp_strategy" + }, + "macro.dbt.split_part": { + "arguments": [], + "created_at": 1689766376.560454, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__split_part" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", + "meta": {}, + "name": "split_part", + "original_file_path": "macros/utils/split_part.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/split_part.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.split_part" + }, + "macro.dbt.sql_convert_columns_in_relation": { + "arguments": [], + "created_at": 1689766376.5897841, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", + "meta": {}, + "name": "sql_convert_columns_in_relation", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.sql_convert_columns_in_relation" + }, + "macro.dbt.statement": { + "arguments": [], + "created_at": 1689766376.542756, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", + "meta": {}, + "name": "statement", + "original_file_path": "macros/etc/statement.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/statement.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.statement" + }, + "macro.dbt.strategy_dispatch": { + "arguments": [], + "created_at": 1689766376.452245, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", + "meta": {}, + "name": "strategy_dispatch", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.strategy_dispatch" + }, + "macro.dbt.string_literal": { + "arguments": [], + "created_at": 1689766376.55523, + "depends_on": { + "macros": [ + "macro.dbt.default__string_literal" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "string_literal", + "original_file_path": "macros/utils/literal.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/literal.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.string_literal" + }, + "macro.dbt.support_multiple_grantees_per_dcl_statement": { + "arguments": [], + "created_at": 1689766376.576416, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "support_multiple_grantees_per_dcl_statement", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement" + }, + "macro.dbt.sync_column_schemas": { + "arguments": [], + "created_at": 1689766376.507675, + "depends_on": { + "macros": [ + "macro.dbt.alter_relation_add_remove_columns", + "macro.dbt.alter_column_type" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", + "meta": {}, + "name": "sync_column_schemas", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.sync_column_schemas" + }, + "macro.dbt.table_columns_and_constraints": { + "arguments": [], + "created_at": 1689766376.510255, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", + "meta": {}, + "name": "table_columns_and_constraints", + "original_file_path": "macros/materializations/models/table/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.table_columns_and_constraints" + }, + "macro.dbt.test_accepted_values": { + "arguments": [], + "created_at": 1689766376.5992289, + "depends_on": { + "macros": [ + "macro.dbt.default__test_accepted_values" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", + "meta": {}, + "name": "test_accepted_values", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_accepted_values" + }, + "macro.dbt.test_not_null": { + "arguments": [], + "created_at": 1689766376.5989408, + "depends_on": { + "macros": [ + "macro.dbt.default__test_not_null" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", + "meta": {}, + "name": "test_not_null", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_not_null" + }, + "macro.dbt.test_relationships": { + "arguments": [], + "created_at": 1689766376.599502, + "depends_on": { + "macros": [ + "macro.dbt.default__test_relationships" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", + "meta": {}, + "name": "test_relationships", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_relationships" + }, + "macro.dbt.test_unique": { + "arguments": [], + "created_at": 1689766376.5987048, + "depends_on": { + "macros": [ + "macro.dbt.default__test_unique" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", + "meta": {}, + "name": "test_unique", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_unique" + }, + "macro.dbt.truncate_relation": { + "arguments": [], + "created_at": 1689766376.571404, + "depends_on": { + "macros": [ + "macro.dbt.default__truncate_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "truncate_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.truncate_relation" + }, + "macro.dbt.type_bigint": { + "arguments": [], + "created_at": 1689766376.5576189, + "depends_on": { + "macros": [ + "macro.dbt.default__type_bigint" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_bigint", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_bigint" + }, + "macro.dbt.type_boolean": { + "arguments": [], + "created_at": 1689766376.558244, + "depends_on": { + "macros": [ + "macro.dbt.default__type_boolean" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_boolean", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_boolean" + }, + "macro.dbt.type_float": { + "arguments": [], + "created_at": 1689766376.556957, + "depends_on": { + "macros": [ + "macro.dbt.default__type_float" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_float", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_float" + }, + "macro.dbt.type_int": { + "arguments": [], + "created_at": 1689766376.557935, + "depends_on": { + "macros": [ + "macro.dbt.default__type_int" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_int", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_int" + }, + "macro.dbt.type_numeric": { + "arguments": [], + "created_at": 1689766376.557278, + "depends_on": { + "macros": [ + "macro.dbt.default__type_numeric" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_numeric", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_numeric" + }, + "macro.dbt.type_string": { + "arguments": [], + "created_at": 1689766376.55626, + "depends_on": { + "macros": [ + "macro.dbt.default__type_string" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_string", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_string" + }, + "macro.dbt.type_timestamp": { + "arguments": [], + "created_at": 1689766376.5566442, + "depends_on": { + "macros": [ + "macro.dbt.default__type_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_timestamp", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_timestamp" + }, + "macro.dbt_databricks.alter_column_set_constraints": { + "arguments": [], + "created_at": 1689766376.275674, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__alter_column_set_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_column_set_constraints", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.alter_column_set_constraints" + }, + "macro.dbt_databricks.alter_table_add_constraints": { + "arguments": [], + "created_at": 1689766376.273877, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__alter_table_add_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_table_add_constraints", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.alter_table_add_constraints" + }, + "macro.dbt_databricks.current_catalog": { + "arguments": [], + "created_at": 1689766376.24827, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__current_catalog" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro current_catalog() -%}\n {{ return(adapter.dispatch('current_catalog', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "current_catalog", + "original_file_path": "macros/catalog.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/catalog.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.current_catalog" + }, + "macro.dbt_databricks.databricks__alter_column_comment": { + "arguments": [], + "created_at": 1689766376.272466, + "depends_on": { + "macros": [ + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__alter_column_comment(relation, column_dict) %}\n {% if config.get('file_format', default='delta') in ['delta', 'hudi'] %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set comment_query %}\n alter table {{ relation }} change column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% endset %}\n {% do run_query(comment_query) %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "databricks__alter_column_comment", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__alter_column_comment" + }, + "macro.dbt_databricks.databricks__alter_column_set_constraints": { + "arguments": [], + "created_at": 1689766376.276233, + "depends_on": { + "macros": [ + "macro.dbt_databricks.get_column_constraints", + "macro.dbt_databricks.get_constraints_sql", + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__alter_column_set_constraints(relation, model) %}\n {% set column_dict = model.columns %}\n {% for column_name in column_dict %}\n {% set column = column_dict[column_name] %}\n {% set constraints = get_column_constraints(column) %}\n {% set statements = get_constraints_sql(relation, constraints, model, column) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "databricks__alter_column_set_constraints", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__alter_column_set_constraints" + }, + "macro.dbt_databricks.databricks__alter_table_add_constraints": { + "arguments": [], + "created_at": 1689766376.27426, + "depends_on": { + "macros": [ + "macro.dbt_databricks.get_model_constraints", + "macro.dbt_databricks.get_constraints_sql", + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__alter_table_add_constraints(relation, model) %}\n {% set constraints = get_model_constraints(model) %}\n {% set statements = get_constraints_sql(relation, constraints, model) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "databricks__alter_table_add_constraints", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__alter_table_add_constraints" + }, + "macro.dbt_databricks.databricks__create_csv_table": { + "arguments": [], + "created_at": 1689766376.329921, + "depends_on": { + "macros": [ + "macro.dbt_spark.file_format_clause", + "macro.dbt_spark.partition_cols", + "macro.dbt_spark.clustered_cols", + "macro.dbt_spark.location_clause", + "macro.dbt_spark.comment_clause", + "macro.dbt_databricks.tblproperties_clause", + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", + "meta": {}, + "name": "databricks__create_csv_table", + "original_file_path": "macros/materializations/seed.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/seed.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__create_csv_table" + }, + "macro.dbt_databricks.databricks__create_table_as": { + "arguments": [], + "created_at": 1689766376.2713711, + "depends_on": { + "macros": [ + "macro.dbt_spark.create_temporary_view", + "macro.dbt.get_assert_columns_equivalent", + "macro.dbt.get_select_subquery", + "macro.dbt_spark.file_format_clause", + "macro.dbt_spark.options_clause", + "macro.dbt_spark.partition_cols", + "macro.dbt_spark.clustered_cols", + "macro.dbt_spark.location_clause", + "macro.dbt_spark.comment_clause", + "macro.dbt_databricks.tblproperties_clause", + "macro.dbt_spark.py_write_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {% if config.get('file_format', default='delta') == 'delta' %}\n create or replace table {{ relation }}\n {% else %}\n create table {{ relation }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause() }}\n {{ options_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "databricks__create_table_as", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__create_table_as" + }, + "macro.dbt_databricks.databricks__create_view_as": { + "arguments": [], + "created_at": 1689766376.271743, + "depends_on": { + "macros": [ + "macro.dbt_spark.comment_clause", + "macro.dbt.get_assert_columns_equivalent", + "macro.dbt_databricks.tblproperties_clause" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__create_view_as(relation, sql) -%}\n create or replace view {{ relation }}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n {{ tblproperties_clause() }}\n as\n {{ sql }}\n{% endmacro %}", + "meta": {}, + "name": "databricks__create_view_as", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__create_view_as" + }, + "macro.dbt_databricks.databricks__current_catalog": { + "arguments": [], + "created_at": 1689766376.248533, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__current_catalog() -%}\n {% call statement('current_catalog', fetch_result=True) %}\n select current_catalog()\n {% endcall %}\n {% do return(load_result('current_catalog').table) %}\n{% endmacro %}", + "meta": {}, + "name": "databricks__current_catalog", + "original_file_path": "macros/catalog.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/catalog.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__current_catalog" + }, + "macro.dbt_databricks.databricks__dateadd": { + "arguments": [], + "created_at": 1689766376.357805, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__dateadd" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__dateadd(datepart, interval, from_date_or_timestamp) %}\n {%- if adapter.compare_dbr_version(10, 4) >= 0 -%}\n timestampadd({{datepart}}, {{interval}}, {{from_date_or_timestamp}})\n {%- else -%}\n {{ spark__dateadd(datepart, interval, from_date_or_timestamp) }}\n {%- endif -%}\n{%- endmacro %}", + "meta": {}, + "name": "databricks__dateadd", + "original_file_path": "macros/utils/dateadd.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/utils/dateadd.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__dateadd" + }, + "macro.dbt_databricks.databricks__datediff": { + "arguments": [], + "created_at": 1689766376.358366, + "depends_on": { + "macros": [ + "macro.dbt.date_trunc", + "macro.dbt_spark.spark__datediff" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__datediff(first_date, second_date, datepart) %}\n {%- if adapter.compare_dbr_version(10, 4) >= 0 -%}\n timestampdiff({{datepart}}, {{date_trunc(datepart, first_date)}}, {{date_trunc(datepart, second_date)}})\n {%- else -%}\n {{ spark__datediff(first_date, second_date, datepart) }}\n {%- endif -%}\n{%- endmacro %}", + "meta": {}, + "name": "databricks__datediff", + "original_file_path": "macros/utils/datediff.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/utils/datediff.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__datediff" + }, + "macro.dbt_databricks.databricks__file_format_clause": { + "arguments": [], + "created_at": 1689766376.268569, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__file_format_clause() %}\n {%- set file_format = config.get('file_format', default='delta') -%}\n {%- if file_format is not none %}\n using {{ file_format }}\n {%- endif %}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "databricks__file_format_clause", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__file_format_clause" + }, + "macro.dbt_databricks.databricks__generate_database_name": { + "arguments": [], + "created_at": 1689766376.287245, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n {{ return(default_database) }}\n {%- else -%}\n {{ return(custom_database_name) }}\n {%- endif -%}\n{%- endmacro %}", + "meta": {}, + "name": "databricks__generate_database_name", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__generate_database_name" + }, + "macro.dbt_databricks.databricks__get_binding_char": { + "arguments": [], + "created_at": 1689766376.290329, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", + "meta": {}, + "name": "databricks__get_binding_char", + "original_file_path": "macros/materializations/seed.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/seed.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__get_binding_char" + }, + "macro.dbt_databricks.databricks__get_incremental_append_sql": { + "arguments": [], + "created_at": 1689766376.349152, + "depends_on": { + "macros": [ + "macro.dbt_databricks.get_insert_into_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__get_incremental_append_sql(arg_dict) %}\n {% do return(get_insert_into_sql(arg_dict[\"temp_relation\"], arg_dict[\"target_relation\"])) %}\n{% endmacro %}", + "meta": {}, + "name": "databricks__get_incremental_append_sql", + "original_file_path": "macros/materializations/incremental/strategies.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__get_incremental_append_sql" + }, + "macro.dbt_databricks.databricks__get_incremental_default_sql": { + "arguments": [], + "created_at": 1689766376.348926, + "depends_on": { + "macros": [ + "macro.dbt.get_incremental_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__get_incremental_default_sql(arg_dict) %}\n {{ return(get_incremental_merge_sql(arg_dict)) }}\n{% endmacro %}", + "meta": {}, + "name": "databricks__get_incremental_default_sql", + "original_file_path": "macros/materializations/incremental/strategies.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__get_incremental_default_sql" + }, + "macro.dbt_databricks.databricks__get_incremental_replace_where_sql": { + "arguments": [], + "created_at": 1689766376.3493152, + "depends_on": { + "macros": [ + "macro.dbt_databricks.get_replace_where_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__get_incremental_replace_where_sql(arg_dict) %}\n {% do return(get_replace_where_sql(arg_dict)) %}\n{% endmacro %}", + "meta": {}, + "name": "databricks__get_incremental_replace_where_sql", + "original_file_path": "macros/materializations/incremental/strategies.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__get_incremental_replace_where_sql" + }, + "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql": { + "arguments": [], + "created_at": 1689766376.349725, + "depends_on": { + "macros": [ + "macro.dbt_databricks.get_insert_overwrite_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) %}\n {{ return(get_insert_overwrite_sql(source, target)) }}\n{% endmacro %}", + "meta": {}, + "name": "databricks__get_insert_overwrite_merge_sql", + "original_file_path": "macros/materializations/incremental/strategies.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql" + }, + "macro.dbt_databricks.databricks__get_merge_sql": { + "arguments": [], + "created_at": 1689766376.352514, + "depends_on": { + "macros": [ + "macro.dbt.get_merge_update_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n when matched then update set\n {% if update_columns -%}{%- for column_name in update_columns %}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {%- else %} * {% endif %}\n\n when not matched then insert *\n{% endmacro %}", + "meta": {}, + "name": "databricks__get_merge_sql", + "original_file_path": "macros/materializations/incremental/strategies.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__get_merge_sql" + }, + "macro.dbt_databricks.databricks__get_or_create_relation": { + "arguments": [], + "created_at": 1689766376.28854, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__get_or_create_relation(database, schema, identifier, type, needs_information=False) %}\n {%- set target_relation = adapter.get_relation(\n database=database,\n schema=schema,\n identifier=identifier,\n needs_information=needs_information) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", + "meta": {}, + "name": "databricks__get_or_create_relation", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__get_or_create_relation" + }, + "macro.dbt_databricks.databricks__list_relations_without_caching": { + "arguments": [], + "created_at": 1689766376.2855082, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__list_relations_without_caching(schema_relation) %}\n {{ return(adapter.get_relations_without_caching(schema_relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "databricks__list_relations_without_caching", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__list_relations_without_caching" + }, + "macro.dbt_databricks.databricks__load_csv_rows": { + "arguments": [], + "created_at": 1689766376.328704, + "depends_on": { + "macros": [ + "macro.dbt.get_batch_size", + "macro.dbt.get_binding_char" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert {% if loop.index0 == 0 -%} overwrite {% else -%} into {% endif -%} {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True, close_cursor=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", + "meta": {}, + "name": "databricks__load_csv_rows", + "original_file_path": "macros/materializations/seed.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/seed.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__load_csv_rows" + }, + "macro.dbt_databricks.databricks__make_temp_relation": { + "arguments": [], + "created_at": 1689766376.2878668, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__make_temp_relation(base_relation, suffix='__dbt_tmp', as_table=False) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {%- if as_table -%}\n {% set tmp_relation = api.Relation.create(\n identifier=tmp_identifier,\n schema=base_relation.schema,\n database=base_relation.database,\n type='table') %}\n {%- else -%}\n {% set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='view') %}\n {%- endif -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", + "meta": {}, + "name": "databricks__make_temp_relation", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__make_temp_relation" + }, + "macro.dbt_databricks.databricks__optimize": { + "arguments": [], + "created_at": 1689766376.2847412, + "depends_on": { + "macros": [ + "macro.dbt.statement", + "macro.dbt_databricks.get_optimize_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__optimize(relation) %}\n {% if config.get('zorder', False) and config.get('file_format', 'delta') == 'delta' %}\n {% if var('DATABRICKS_SKIP_OPTIMIZE', 'false')|lower != 'true' and var('databricks_skip_optimize', 'false')|lower != 'true' %}\n {% call statement('run_optimize_stmt') %}\n {{ get_optimize_sql(relation) }}\n {% endcall %}\n {% endif %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "databricks__optimize", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__optimize" + }, + "macro.dbt_databricks.databricks__options_clause": { + "arguments": [], + "created_at": 1689766376.2697349, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__options_clause() -%}\n {%- set options = config.get('options') -%}\n {%- if config.get('file_format', default='delta') == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "databricks__options_clause", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__options_clause" + }, + "macro.dbt_databricks.databricks__persist_constraints": { + "arguments": [], + "created_at": 1689766376.273657, + "depends_on": { + "macros": [ + "macro.dbt.is_incremental", + "macro.dbt_databricks.alter_table_add_constraints", + "macro.dbt_databricks.alter_column_set_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% set has_model_contract = contract_config and contract_config.enforced %}\n {% set has_databricks_constraints = config.get('persist_constraints', False) %}\n\n {% if (has_model_contract or has_databricks_constraints) %}\n {% if config.get('file_format', 'delta') != 'delta' %}\n {# Constraints are only supported for delta tables #}\n {{ exceptions.warn(\"Constraints not supported for file format: \" ~ config.get('file_format')) }}\n {% elif relation.is_view %}\n {# Constraints are not supported for views. This point in the code should not have been reached. #}\n {{ exceptions.raise_compiler_error(\"Constraints not supported for views.\") }}\n {% elif is_incremental() %}\n {# Constraints are not applied for incremental updates. This point in the code should not have been reached #}\n {{ exceptions.raise_compiler_error(\"Constraints are not applied for incremental updates. Full refresh is required to update constraints.\") }}\n {% else %}\n {% do alter_table_add_constraints(relation, model) %}\n {% do alter_column_set_constraints(relation, model) %}\n {% endif %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "databricks__persist_constraints", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__persist_constraints" + }, + "macro.dbt_databricks.databricks__show_table_extended": { + "arguments": [], + "created_at": 1689766376.286025, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__show_table_extended(schema_relation) %}\n {% call statement('show_table_extended', fetch_result=True) -%}\n show table extended in {{ schema_relation.without_identifier() }} like '{{ schema_relation.identifier }}'\n {% endcall %}\n\n {% do return(load_result('show_table_extended').table) %}\n{% endmacro %}", + "meta": {}, + "name": "databricks__show_table_extended", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__show_table_extended" + }, + "macro.dbt_databricks.databricks__show_tables": { + "arguments": [], + "created_at": 1689766376.2865, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__show_tables(relation) %}\n {% call statement('show_tables', fetch_result=True) -%}\n show tables in {{ relation }}\n {% endcall %}\n\n {% do return(load_result('show_tables').table) %}\n{% endmacro %}", + "meta": {}, + "name": "databricks__show_tables", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__show_tables" + }, + "macro.dbt_databricks.databricks__show_views": { + "arguments": [], + "created_at": 1689766376.2869499, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__show_views(relation) %}\n {% call statement('show_views', fetch_result=True) -%}\n show views in {{ relation }}\n {% endcall %}\n\n {% do return(load_result('show_views').table) %}\n{% endmacro %}", + "meta": {}, + "name": "databricks__show_views", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__show_views" + }, + "macro.dbt_databricks.databricks__tblproperties_clause": { + "arguments": [], + "created_at": 1689766376.2703009, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__tblproperties_clause() -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- if tblproperties is not none %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "databricks__tblproperties_clause", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__tblproperties_clause" + }, + "macro.dbt_databricks.databricks__use_catalog": { + "arguments": [], + "created_at": 1689766376.2489321, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks__use_catalog(catalog) -%}\n {% call statement() %}\n use catalog {{ adapter.quote(catalog) }}\n {% endcall %}\n{% endmacro %}", + "meta": {}, + "name": "databricks__use_catalog", + "original_file_path": "macros/catalog.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/catalog.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks__use_catalog" + }, + "macro.dbt_databricks.databricks_build_snapshot_staging_table": { + "arguments": [], + "created_at": 1689766376.339793, + "depends_on": { + "macros": [ + "macro.dbt.snapshot_staging_table", + "macro.dbt.statement", + "macro.dbt.create_view_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_identifier = target_relation.identifier ~ '__dbt_tmp' %}\n\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=target_relation.schema,\n database=target_relation.database,\n type='view') -%}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_view_as(tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", + "meta": {}, + "name": "databricks_build_snapshot_staging_table", + "original_file_path": "macros/materializations/snapshot.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/snapshot.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks_build_snapshot_staging_table" + }, + "macro.dbt_databricks.databricks_constraints_to_dbt": { + "arguments": [], + "created_at": 1689766376.284013, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks_constraints_to_dbt(constraints, column) %}\n {# convert constraints defined using the original databricks format #}\n {% set dbt_constraints = [] %}\n {% for constraint in constraints %}\n {% if constraint.get and constraint.get(\"type\") %}\n {# already in model contract format #}\n {% do dbt_constraints.append(constraint) %}\n {% else %}\n {% if column %}\n {% if constraint == \"not_null\" %}\n {% do dbt_constraints.append({\"type\": \"not_null\", \"columns\": [column.get(\"name\")]}) %}\n {% else %}\n {{ exceptions.raise_compiler_error('Invalid constraint for column ' ~ column.get(\"name\", \"\") ~ '. Only `not_null` is supported.') }}\n {% endif %}\n {% else %}\n {% set name = constraint['name'] %}\n {% if not name %}\n {{ exceptions.raise_compiler_error('Invalid check constraint name') }}\n {% endif %}\n {% set condition = constraint['condition'] %}\n {% if not condition %}\n {{ exceptions.raise_compiler_error('Invalid check constraint condition') }}\n {% endif %}\n {% do dbt_constraints.append({\"name\": name, \"type\": \"check\", \"expression\": condition}) %}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(dbt_constraints) }}\n{% endmacro %}", + "meta": {}, + "name": "databricks_constraints_to_dbt", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks_constraints_to_dbt" + }, + "macro.dbt_databricks.databricks_copy_into": { + "arguments": [], + "created_at": 1689766376.247835, + "depends_on": { + "macros": [ + "macro.dbt.get_or_create_relation", + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro databricks_copy_into(\n target_table,\n source,\n file_format,\n expression_list=none,\n source_credential=none,\n source_encryption=none,\n validate=none,\n files=none,\n pattern=none,\n format_options=none,\n copy_options=none) -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=target.database,\n schema=target.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- set source_clause -%}\n {%- if expression_list -%}\n ( select {{ expression_list }} from '{{ source }}' )\n {%- else -%}\n '{{ source }}'\n {%- endif -%}\n {%- if source_credential or source_encryption %}\n WITH (\n {%- if source_credential %}\n credential (\n {%- for name in source_credential -%}\n '{{ name }}' = '{{ source_credential[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {%- if source_encryption %}\n encryption (\n {%- for name in source_encryption -%}\n '{{ name }}' = '{{ source_encryption[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n )\n {%- endif -%}\n {%- endset -%}\n\n {% set query %}\n copy into {{ target_relation }}\n from {{ source_clause }}\n fileformat = {{ file_format }}\n {% if validate -%} validate {{ validate }} {%- endif %}\n {% if files and pattern %}\n {{ exceptions.raise_compiler_error(\"You can only specify one of 'files' or 'pattern'\") }}\n {% endif %}\n {% if files -%}\n files = (\n {%- for file in files -%}\n '{{ file }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if pattern -%}\n pattern = '{{ pattern }}'\n {%- endif %}\n {% if format_options -%}\n format_options (\n {%- for key in format_options -%}\n '{{ key }}' = '{{ format_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if copy_options -%}\n copy_options (\n {%- for key in copy_options -%}\n '{{ key }}' = '{{ copy_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% endset %}\n\n {% do log(\"Running COPY INTO\" ~ adapter.redact_credentials(query), info=True) %}\n {% do run_query(query) %}\n\n{% endmacro %}", + "meta": {}, + "name": "databricks_copy_into", + "original_file_path": "macros/copy_into.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/copy_into.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.databricks_copy_into" + }, + "macro.dbt_databricks.dbt_databricks_validate_get_file_format": { + "arguments": [], + "created_at": 1689766376.3450959, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro dbt_databricks_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", + "meta": {}, + "name": "dbt_databricks_validate_get_file_format", + "original_file_path": "macros/materializations/incremental/validate.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/incremental/validate.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_file_format" + }, + "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy": { + "arguments": [], + "created_at": 1689766376.3461492, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n Expected one of: 'merge', 'replace_where', 'append', 'insert_overwrite'\n {%- endset %}\n\n {% set invalid_delta_only_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via warehouse\n Use the 'merge' or 'replace_where' strategy instead\n {%- endset %}\n\n {% if raw_strategy not in ['append', 'merge', 'insert_overwrite', 'replace_where'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% if raw_strategy == 'replace_where' and file_format not in ['delta'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% if raw_strategy == 'insert_overwrite' and target.endpoint %}\n {% do exceptions.raise_compiler_error(invalid_insert_overwrite_endpoint_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", + "meta": {}, + "name": "dbt_databricks_validate_get_incremental_strategy", + "original_file_path": "macros/materializations/incremental/validate.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/incremental/validate.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy" + }, + "macro.dbt_databricks.get_column_constraints": { + "arguments": [], + "created_at": 1689766376.2754612, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks_constraints_to_dbt" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_column_constraints(column) %}\n {% set constraints = column.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and column.get('meta', {}).get('constraint') %}\n {# Databricks constraints implementation. Constraint is in the meta property. #}\n {% set db_constraints = [column.get('meta', {}).get('constraint')] %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints, column) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", + "meta": {}, + "name": "get_column_constraints", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.get_column_constraints" + }, + "macro.dbt_databricks.get_constraint_sql": { + "arguments": [], + "created_at": 1689766376.2826629, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_constraint_sql(relation, constraint, model, column={}) %}\n {% set statements = [] %}\n {% set type = constraint.get(\"type\", \"\") %}\n\n {% if type == 'check' %}\n {% set expression = constraint.get(\"expression\", \"\") %}\n {% if not expression %}\n {{ exceptions.raise_compiler_error('Invalid check constraint expression') }}\n {% endif %}\n\n {% set name = constraint.get(\"name\") %}\n {% if not name and local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead.\") }}\n {%- set name = local_md5 (column.get(\"name\", \"\") ~ \";\" ~ expression ~ \";\") -%}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" check (\" ~ expression ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'not_null' %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if column %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% set stmt = \"alter table \" ~ relation ~ \" change column \" ~ quoted_name ~ \" set not null \" ~ (constraint.expression or \"\") ~ \";\" %}\n {% do statements.append(stmt) %}\n {% else %}\n {{ exceptions.warn('not_null constraint on invalid column: ' ~ column_name) }}\n {% endif %}\n {% endfor %}\n {% elif type == 'primary_key' %}\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ type)}}\n {% endif %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid primary key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set name = constraint.get(\"name\") %}\n {% if not name and local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead.\") }}\n {%- set name = local_md5(\"primary_key;\" ~ column_names ~ \";\") -%}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" primary key(\" ~ joined_names ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'foreign_key' %}\n\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid foreign key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set name = constraint.get(\"name\") %}\n {% if not name and local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead.\") }}\n {%- set name = local_md5(\"primary_key;\" ~ column_names ~ \";\") -%}\n {% endif %}\n\n {% set parent = constraint.get(\"parent\") %}\n {% if not parent %} \n {{ exceptions.raise_compiler_error('No parent table defined for foreign key: ' ~ expression) }}\n {% endif %}\n {% if not \".\" in parent %}\n {% set parent = relation.schema ~ \".\" ~ parent%}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" foreign key(\" ~ joined_names ~ \") references \" ~ parent %}\n {% set parent_columns = constraint.get(\"parent_columns\") %}\n {% if parent_columns %}\n {% set stmt = stmt ~ \"(\" ~ parent_columns|join(\", \") ~ \")\"%}\n {% endif %}\n {% set stmt = stmt ~ \";\" %}\n {% do statements.append(stmt) %}\n {% elif constraint.get('warn_unsupported') %}\n {{ exceptions.warn(\"unsupported constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {{ return(statements) }}\n{% endmacro %}", + "meta": {}, + "name": "get_constraint_sql", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.get_constraint_sql" + }, + "macro.dbt_databricks.get_constraints_sql": { + "arguments": [], + "created_at": 1689766376.2767968, + "depends_on": { + "macros": [ + "macro.dbt_databricks.get_constraint_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_constraints_sql(relation, constraints, model, column) %}\n {% set statements = [] %}\n {% for constraint in constraints %}\n {% if constraint %}\n {% set constraint_statements = get_constraint_sql(relation, constraint, model, column) %}\n {% for statement in constraint_statements %}\n {% if statement %}\n {% do statements.append(statement) %}\n {% endif %}\n {% endfor %} \n {% endif %}\n {% endfor %}\n\n {{ return(statements) }}\n{% endmacro %}", + "meta": {}, + "name": "get_constraints_sql", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.get_constraints_sql" + }, + "macro.dbt_databricks.get_incremental_replace_where_sql": { + "arguments": [], + "created_at": 1689766376.34952, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__get_incremental_replace_where_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_replace_where_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_replace_where_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_replace_where_sql", + "original_file_path": "macros/materializations/incremental/strategies.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.get_incremental_replace_where_sql" + }, + "macro.dbt_databricks.get_insert_into_sql": { + "arguments": [], + "created_at": 1689766376.350976, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert into table {{ target_relation }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_insert_into_sql", + "original_file_path": "macros/materializations/incremental/strategies.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.get_insert_into_sql" + }, + "macro.dbt_databricks.get_insert_overwrite_sql": { + "arguments": [], + "created_at": 1689766376.3501089, + "depends_on": { + "macros": [ + "macro.dbt_spark.partition_cols" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_insert_overwrite_sql", + "original_file_path": "macros/materializations/incremental/strategies.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.get_insert_overwrite_sql" + }, + "macro.dbt_databricks.get_model_constraints": { + "arguments": [], + "created_at": 1689766376.274861, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks_constraints_to_dbt" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_model_constraints(model) %}\n {% set constraints = model.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and model.get('meta', {}).get('constraints') is sequence %}\n {# Databricks constraints implementation. Constraints are in the meta property. #}\n {% set db_constraints = model.get('meta', {}).get('constraints', []) %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", + "meta": {}, + "name": "get_model_constraints", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.get_model_constraints" + }, + "macro.dbt_databricks.get_optimize_sql": { + "arguments": [], + "created_at": 1689766376.2853482, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_optimize_sql(relation) %}\n {% if config.get('zorder', False) and config.get('file_format', 'delta') == 'delta' %}\n {%- set zorder = config.get('zorder', none) -%}\n optimize {{ relation }}\n {# TODO: predicates here? WHERE ... #}\n {% if zorder is sequence and zorder is not string %}\n zorder by (\n {%- for col in zorder -%}\n {{ col }}{% if not loop.last %}, {% endif %}\n {%- endfor -%}\n )\n {% else %}\n zorder by ({{zorder}})\n {% endif %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "get_optimize_sql", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.get_optimize_sql" + }, + "macro.dbt_databricks.get_replace_where_sql": { + "arguments": [], + "created_at": 1689766376.35065, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_replace_where_sql(args_dict) -%}\n {%- set predicates = args_dict['incremental_predicates'] -%}\n {%- set target_relation = args_dict['target_relation'] -%}\n {%- set temp_relation = args_dict['temp_relation'] -%}\n\n insert into {{ target_relation }}\n {% if predicates %}\n {% if predicates is sequence and predicates is not string %}\n replace where {{ predicates | join(' and ') }}\n {% else %}\n replace where {{ predicates }}\n {% endif %}\n {% endif %}\n table {{ temp_relation }}\n \n{% endmacro %}", + "meta": {}, + "name": "get_replace_where_sql", + "original_file_path": "macros/materializations/incremental/strategies.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.get_replace_where_sql" + }, + "macro.dbt_databricks.materialization_incremental_databricks": { + "arguments": [], + "created_at": 1689766376.357175, + "depends_on": { + "macros": [ + "macro.dbt_databricks.dbt_databricks_validate_get_file_format", + "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy", + "macro.dbt.incremental_validate_on_schema_change", + "macro.dbt.statement", + "macro.dbt.run_hooks", + "macro.dbt.create_table_as", + "macro.dbt_databricks.persist_constraints", + "macro.dbt.should_full_refresh", + "macro.dbt_databricks.databricks__make_temp_relation", + "macro.dbt.process_schema_changes", + "macro.dbt_databricks.statement_with_staging_table", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt_databricks.optimize" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization incremental, adapter='databricks', supported_languages=['sql', 'python'] -%}\n {#-- Validate early so we don't run SQL if the file_format + strategy combo is invalid --#}\n {%- set raw_file_format = config.get('file_format', default='delta') -%}\n {%- set raw_strategy = config.get('incremental_strategy') or 'merge' -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set file_format = dbt_databricks_validate_get_file_format(raw_file_format) -%}\n {%- set incremental_strategy = dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) -%}\n\n {#-- Set vars --#}\n\n {%- set incremental_predicates = config.get('predicates', default=none) or config.get('incremental_predicates', default=none) -%}\n {%- set unique_key = config.get('unique_key', none) -%}\n {%- set partition_by = config.get('partition_by', none) -%}\n {%- set language = model['language'] -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set target_relation = this -%}\n {%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier, needs_information=True) -%}\n\n {#-- Set Overwrite Mode - does not work for warehouses --#}\n {%- if incremental_strategy == 'insert_overwrite' and partition_by -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = DYNAMIC\n {%- endcall -%}\n {%- endif -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- elif existing_relation.is_view or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %}\n {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% if not existing_relation.is_view %}\n {% do persist_constraints(target_relation, model) %}\n {% endif %}\n {%- else -%}\n {#-- Relation must be merged --#}\n {%- set temp_relation = databricks__make_temp_relation(target_relation, as_table=language != 'sql') -%}\n {%- call statement('create_temp_relation', language=language) -%}\n {{ create_table_as(True, temp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, temp_relation, existing_relation) -%}\n {%- set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) -%}\n {%- set strategy_arg_dict = ({\n 'target_relation': target_relation,\n 'temp_relation': temp_relation,\n 'unique_key': unique_key,\n 'dest_columns': none,\n 'incremental_predicates': incremental_predicates}) -%}\n {%- set build_sql = strategy_sql_macro_func(strategy_arg_dict) -%}\n {%- if language == 'sql' -%}\n {%- call statement('main') -%}\n {{ build_sql }}\n {%- endcall -%}\n {%- elif language == 'python' -%}\n {%- call statement_with_staging_table('main', temp_relation) -%}\n {{ build_sql }}\n {%- endcall -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why does not either drop_relation or adapter.drop_relation work here?!\n --#}\n {%- endif -%}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", + "meta": {}, + "name": "materialization_incremental_databricks", + "original_file_path": "macros/materializations/incremental/incremental.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/incremental/incremental.sql", + "resource_type": "macro", + "supported_languages": [ + "sql", + "python" + ], + "unique_id": "macro.dbt_databricks.materialization_incremental_databricks" + }, + "macro.dbt_databricks.materialization_snapshot_databricks": { + "arguments": [], + "created_at": 1689766376.343799, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__get_or_create_relation", + "macro.dbt.run_hooks", + "macro.dbt.strategy_dispatch", + "macro.dbt.build_snapshot_table", + "macro.dbt.create_table_as", + "macro.dbt.statement", + "macro.dbt_spark.spark_build_snapshot_staging_table", + "macro.dbt_databricks.databricks_build_snapshot_staging_table", + "macro.dbt.create_columns", + "macro.dbt.snapshot_merge_sql", + "macro.dbt_databricks.statement_with_staging_table", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt_databricks.persist_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization snapshot, adapter='databricks' %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = config.get('file_format', 'delta') -%}\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = databricks__get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table',\n needs_information=True) -%}\n\n {%- if file_format not in ['delta', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% if target_relation.database is none %}\n {% set staging_table = spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% else %}\n {% set staging_table = databricks_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% endif %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% call statement_with_staging_table('main', staging_table) %}\n {{ final_sql }}\n {% endcall %}\n\n {% endif %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_snapshot_databricks", + "original_file_path": "macros/materializations/snapshot.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/snapshot.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt_databricks.materialization_snapshot_databricks" + }, + "macro.dbt_databricks.materialization_table_databricks": { + "arguments": [], + "created_at": 1689766376.3322399, + "depends_on": { + "macros": [ + "macro.dbt.run_hooks", + "macro.dbt.statement", + "macro.dbt.create_table_as", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt_databricks.persist_constraints", + "macro.dbt_databricks.optimize" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization table, adapter = 'databricks', supported_languages=['sql', 'python'] %}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if old_relation and not (old_relation.is_delta and config.get('file_format', default='delta') == 'delta') -%}\n {{ adapter.drop_relation(old_relation) }}\n {%- endif %}\n\n -- build model\n\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]})}}\n\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_table_databricks", + "original_file_path": "macros/materializations/table.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/table.sql", + "resource_type": "macro", + "supported_languages": [ + "sql", + "python" + ], + "unique_id": "macro.dbt_databricks.materialization_table_databricks" + }, + "macro.dbt_databricks.materialization_view_databricks": { + "arguments": [], + "created_at": 1689766376.3301492, + "depends_on": { + "macros": [ + "macro.dbt.create_or_replace_view" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization view, adapter='databricks' -%}\n {{ return(create_or_replace_view()) }}\n{%- endmaterialization %}", + "meta": {}, + "name": "materialization_view_databricks", + "original_file_path": "macros/materializations/view.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/materializations/view.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt_databricks.materialization_view_databricks" + }, + "macro.dbt_databricks.optimize": { + "arguments": [], + "created_at": 1689766376.284213, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__optimize" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro optimize(relation) %}\n {{ return(adapter.dispatch('optimize', 'dbt')(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "optimize", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.optimize" + }, + "macro.dbt_databricks.persist_constraints": { + "arguments": [], + "created_at": 1689766376.2726789, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__persist_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", + "meta": {}, + "name": "persist_constraints", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.persist_constraints" + }, + "macro.dbt_databricks.show_table_extended": { + "arguments": [], + "created_at": 1689766376.2856991, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__show_table_extended" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro show_table_extended(schema_relation) %}\n {{ return(adapter.dispatch('show_table_extended', 'dbt')(schema_relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "show_table_extended", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.show_table_extended" + }, + "macro.dbt_databricks.show_tables": { + "arguments": [], + "created_at": 1689766376.286232, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__show_tables" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro show_tables(relation) %}\n {{ return(adapter.dispatch('show_tables', 'dbt')(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "show_tables", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.show_tables" + }, + "macro.dbt_databricks.show_views": { + "arguments": [], + "created_at": 1689766376.2866828, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__show_views" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro show_views(relation) %}\n {{ return(adapter.dispatch('show_views', 'dbt')(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "show_views", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.show_views" + }, + "macro.dbt_databricks.statement_with_staging_table": { + "arguments": [], + "created_at": 1689766376.243063, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro statement_with_staging_table(name=None, staging_table=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result, staging_table=staging_table) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", + "meta": {}, + "name": "statement_with_staging_table", + "original_file_path": "macros/statement.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/statement.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.statement_with_staging_table" + }, + "macro.dbt_databricks.tblproperties_clause": { + "arguments": [], + "created_at": 1689766376.2699149, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__tblproperties_clause" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro tblproperties_clause() -%}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "tblproperties_clause", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.tblproperties_clause" + }, + "macro.dbt_databricks.use_catalog": { + "arguments": [], + "created_at": 1689766376.248739, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__use_catalog" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro use_catalog(catalog) -%}\n {{ return(adapter.dispatch('use_catalog', 'dbt')(catalog)) }}\n{% endmacro %}", + "meta": {}, + "name": "use_catalog", + "original_file_path": "macros/catalog.sql", + "package_name": "dbt_databricks", + "patch_path": null, + "path": "macros/catalog.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_databricks.use_catalog" + }, + "macro.dbt_spark.alter_column_set_constraints": { + "arguments": [], + "created_at": 1689766376.3843, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__alter_column_set_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_column_set_constraints", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.alter_column_set_constraints" + }, + "macro.dbt_spark.alter_table_add_constraints": { + "arguments": [], + "created_at": 1689766376.383418, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__alter_table_add_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_table_add_constraints", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.alter_table_add_constraints" + }, + "macro.dbt_spark.assert_not_null": { + "arguments": [], + "created_at": 1689766376.431045, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__assert_not_null" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro assert_not_null(function, arg) -%}\n {{ return(adapter.dispatch('assert_not_null', 'dbt')(function, arg)) }}\n{%- endmacro %}", + "meta": {}, + "name": "assert_not_null", + "original_file_path": "macros/utils/assert_not_null.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/utils/assert_not_null.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.assert_not_null" + }, + "macro.dbt_spark.clustered_cols": { + "arguments": [], + "created_at": 1689766376.3796961, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__clustered_cols" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro clustered_cols(label, required=false) %}\n {{ return(adapter.dispatch('clustered_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "clustered_cols", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.clustered_cols" + }, + "macro.dbt_spark.comment_clause": { + "arguments": [], + "created_at": 1689766376.378133, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__comment_clause" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro comment_clause() %}\n {{ return(adapter.dispatch('comment_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "comment_clause", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.comment_clause" + }, + "macro.dbt_spark.create_temporary_view": { + "arguments": [], + "created_at": 1689766376.3812082, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__create_temporary_view" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_temporary_view(relation, compiled_code) -%}\n {{ return(adapter.dispatch('create_temporary_view', 'dbt')(relation, compiled_code)) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "create_temporary_view", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.create_temporary_view" + }, + "macro.dbt_spark.dbt_spark_get_incremental_sql": { + "arguments": [], + "created_at": 1689766376.4266958, + "depends_on": { + "macros": [ + "macro.dbt_spark.get_insert_into_sql", + "macro.dbt_spark.get_insert_overwrite_sql", + "macro.dbt.get_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro dbt_spark_get_incremental_sql(strategy, source, target, existing, unique_key, incremental_predicates) %}\n {%- if strategy == 'append' -%}\n {#-- insert new records into existing table, without updating or overwriting #}\n {{ get_insert_into_sql(source, target) }}\n {%- elif strategy == 'insert_overwrite' -%}\n {#-- insert statements don't like CTEs, so support them via a temp view #}\n {{ get_insert_overwrite_sql(source, target, existing) }}\n {%- elif strategy == 'merge' -%}\n {#-- merge all columns for datasources which implement MERGE INTO (e.g. databricks, iceberg) - schema changes are handled for us #}\n {{ get_merge_sql(target, source, unique_key, dest_columns=none, incremental_predicates=incremental_predicates) }}\n {%- else -%}\n {% set no_sql_for_strategy_msg -%}\n No known SQL for the incremental strategy provided: {{ strategy }}\n {%- endset %}\n {%- do exceptions.raise_compiler_error(no_sql_for_strategy_msg) -%}\n {%- endif -%}\n\n{% endmacro %}", + "meta": {}, + "name": "dbt_spark_get_incremental_sql", + "original_file_path": "macros/materializations/incremental/strategies.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.dbt_spark_get_incremental_sql" + }, + "macro.dbt_spark.dbt_spark_tblproperties_clause": { + "arguments": [], + "created_at": 1689766376.375644, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro dbt_spark_tblproperties_clause() -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- if tblproperties is not none %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "dbt_spark_tblproperties_clause", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.dbt_spark_tblproperties_clause" + }, + "macro.dbt_spark.dbt_spark_validate_get_file_format": { + "arguments": [], + "created_at": 1689766376.4208262, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro dbt_spark_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'iceberg', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", + "meta": {}, + "name": "dbt_spark_validate_get_file_format", + "original_file_path": "macros/materializations/incremental/validate.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/incremental/validate.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.dbt_spark_validate_get_file_format" + }, + "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy": { + "arguments": [], + "created_at": 1689766376.4220312, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n Expected one of: 'append', 'merge', 'insert_overwrite'\n {%- endset %}\n\n {% set invalid_merge_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n\n {% set invalid_insert_overwrite_delta_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when file_format is set to 'delta' or 'iceberg'\n Use the 'append' or 'merge' strategy instead\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via endpoint\n Use the 'append' or 'merge' strategy instead\n {%- endset %}\n\n {% if raw_strategy not in ['append', 'merge', 'insert_overwrite'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'iceberg', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_merge_msg) %}\n {% endif %}\n {% if raw_strategy == 'insert_overwrite' and file_format == 'delta' %}\n {% do exceptions.raise_compiler_error(invalid_insert_overwrite_delta_msg) %}\n {% endif %}\n {% if raw_strategy == 'insert_overwrite' and target.endpoint %}\n {% do exceptions.raise_compiler_error(invalid_insert_overwrite_endpoint_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", + "meta": {}, + "name": "dbt_spark_validate_get_incremental_strategy", + "original_file_path": "macros/materializations/incremental/validate.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/incremental/validate.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy" + }, + "macro.dbt_spark.describe_table_extended_without_caching": { + "arguments": [], + "created_at": 1689766376.387445, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro describe_table_extended_without_caching(table_name) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('describe_table_extended_without_caching', fetch_result=True) -%}\n describe extended {{ table_name }}\n {% endcall %}\n {% do return(load_result('describe_table_extended_without_caching').table) %}\n{% endmacro %}", + "meta": {}, + "name": "describe_table_extended_without_caching", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.describe_table_extended_without_caching" + }, + "macro.dbt_spark.fetch_tbl_properties": { + "arguments": [], + "created_at": 1689766376.380939, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro fetch_tbl_properties(relation) -%}\n {% call statement('list_properties', fetch_result=True) -%}\n SHOW TBLPROPERTIES {{ relation }}\n {% endcall %}\n {% do return(load_result('list_properties').table) %}\n{%- endmacro %}", + "meta": {}, + "name": "fetch_tbl_properties", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.fetch_tbl_properties" + }, + "macro.dbt_spark.file_format_clause": { + "arguments": [], + "created_at": 1689766376.3758452, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__file_format_clause" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro file_format_clause() %}\n {{ return(adapter.dispatch('file_format_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "file_format_clause", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.file_format_clause" + }, + "macro.dbt_spark.get_columns_in_relation_raw": { + "arguments": [], + "created_at": 1689766376.385899, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__get_columns_in_relation_raw" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_columns_in_relation_raw(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation_raw', 'dbt')(relation)) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "get_columns_in_relation_raw", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.get_columns_in_relation_raw" + }, + "macro.dbt_spark.get_insert_into_sql": { + "arguments": [], + "created_at": 1689766376.4245381, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert into table {{ target_relation }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_insert_into_sql", + "original_file_path": "macros/materializations/incremental/strategies.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.get_insert_into_sql" + }, + "macro.dbt_spark.get_insert_overwrite_sql": { + "arguments": [], + "created_at": 1689766376.424203, + "depends_on": { + "macros": [ + "macro.dbt_spark.partition_cols" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation, existing_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n {% if existing_relation.is_iceberg %}\n {# removed table from statement for iceberg #}\n insert overwrite {{ target_relation }}\n {# removed partition_cols for iceberg as well #}\n {% else %}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n {% endif %}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_insert_overwrite_sql", + "original_file_path": "macros/materializations/incremental/strategies.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.get_insert_overwrite_sql" + }, + "macro.dbt_spark.list_relations_show_tables_without_caching": { + "arguments": [], + "created_at": 1689766376.387159, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro list_relations_show_tables_without_caching(schema_relation) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('list_relations_without_caching_show_tables', fetch_result=True) -%}\n show tables in {{ schema_relation }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching_show_tables').table) %}\n{% endmacro %}", + "meta": {}, + "name": "list_relations_show_tables_without_caching", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.list_relations_show_tables_without_caching" + }, + "macro.dbt_spark.location_clause": { + "arguments": [], + "created_at": 1689766376.376299, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__location_clause" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro location_clause() %}\n {{ return(adapter.dispatch('location_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "location_clause", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.location_clause" + }, + "macro.dbt_spark.materialization_incremental_spark": { + "arguments": [], + "created_at": 1689766376.430502, + "depends_on": { + "macros": [ + "macro.dbt_spark.dbt_spark_validate_get_file_format", + "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", + "macro.dbt.incremental_validate_on_schema_change", + "macro.dbt.load_relation", + "macro.dbt.make_temp_relation", + "macro.dbt.statement", + "macro.dbt.run_hooks", + "macro.dbt.create_table_as", + "macro.dbt_spark.persist_constraints", + "macro.dbt.should_full_refresh", + "macro.dbt.process_schema_changes", + "macro.dbt_spark.dbt_spark_get_incremental_sql", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization incremental, adapter='spark', supported_languages=['sql', 'python'] -%}\n {#-- Validate early so we don't run SQL if the file_format + strategy combo is invalid --#}\n {%- set raw_file_format = config.get('file_format', default='parquet') -%}\n {%- set raw_strategy = config.get('incremental_strategy') or 'append' -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set file_format = dbt_spark_validate_get_file_format(raw_file_format) -%}\n {%- set strategy = dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) -%}\n\n {#-- Set vars --#}\n\n {%- set unique_key = config.get('unique_key', none) -%}\n {%- set partition_by = config.get('partition_by', none) -%}\n {%- set language = model['language'] -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%}\n {%- set target_relation = this -%}\n {%- set existing_relation = load_relation(this) -%}\n {%- set tmp_relation = make_temp_relation(this) -%}\n\n {#-- for SQL model we will create temp view that doesn't have database and schema --#}\n {%- if language == 'sql'-%}\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {%- endif -%}\n\n {#-- Set Overwrite Mode --#}\n {%- if strategy == 'insert_overwrite' and partition_by -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = DYNAMIC\n {%- endcall -%}\n {%- endif -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- elif existing_relation.is_view or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %}\n {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- else -%}\n {#-- Relation must be merged --#}\n {%- call statement('create_tmp_relation', language=language) -%}\n {{ create_table_as(True, tmp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, tmp_relation, existing_relation) -%}\n {%- call statement('main') -%}\n {{ dbt_spark_get_incremental_sql(strategy, tmp_relation, target_relation, existing_relation, unique_key, incremental_predicates) }}\n {%- endcall -%}\n {%- if language == 'python' -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why do neither drop_relation or adapter.drop_relation work here?!\n --#}\n {% call statement('drop_relation') -%}\n drop table if exists {{ tmp_relation }}\n {%- endcall %}\n {%- endif -%}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", + "meta": {}, + "name": "materialization_incremental_spark", + "original_file_path": "macros/materializations/incremental/incremental.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/incremental/incremental.sql", + "resource_type": "macro", + "supported_languages": [ + "sql", + "python" + ], + "unique_id": "macro.dbt_spark.materialization_incremental_spark" + }, + "macro.dbt_spark.materialization_snapshot_spark": { + "arguments": [], + "created_at": 1689766376.418359, + "depends_on": { + "macros": [ + "macro.dbt.get_or_create_relation", + "macro.dbt.create_schema", + "macro.dbt.run_hooks", + "macro.dbt.strategy_dispatch", + "macro.dbt.build_snapshot_table", + "macro.dbt.create_table_as", + "macro.dbt_spark.spark_build_snapshot_staging_table", + "macro.dbt.create_columns", + "macro.dbt.snapshot_merge_sql", + "macro.dbt.statement", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt.post_snapshot" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization snapshot, adapter='spark' %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = config.get('file_format', 'parquet') -%}\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=none,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if file_format not in ['delta', 'iceberg', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_iceberg and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {% if not adapter.check_schema_exists(model.database, model.schema) %}\n {% do create_schema(model.schema) %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_snapshot_spark", + "original_file_path": "macros/materializations/snapshot.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/snapshot.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt_spark.materialization_snapshot_spark" + }, + "macro.dbt_spark.materialization_table_spark": { + "arguments": [], + "created_at": 1689766376.4027252, + "depends_on": { + "macros": [ + "macro.dbt.run_hooks", + "macro.dbt.statement", + "macro.dbt.create_table_as", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt_spark.persist_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization table, adapter = 'spark', supported_languages=['sql', 'python'] %}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta or iceberg, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if old_relation is not none %}\n {% set is_delta = (old_relation.is_delta and config.get('file_format', validator=validation.any[basestring]) == 'delta') %}\n {% set is_iceberg = (old_relation.is_iceberg and config.get('file_format', validator=validation.any[basestring]) == 'iceberg') %}\n {% set old_relation_type = old_relation.type %}\n {% else %}\n {% set is_delta = false %}\n {% set is_iceberg = false %}\n {% set old_relation_type = target_relation.type %}\n {% endif %}\n\n {% if not is_delta and not is_iceberg %}\n {% set existing_relation = target_relation %}\n {{ adapter.drop_relation(existing_relation.incorporate(type=old_relation_type)) }}\n {% endif %}\n\n -- build model\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]})}}\n\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_table_spark", + "original_file_path": "macros/materializations/table.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/table.sql", + "resource_type": "macro", + "supported_languages": [ + "sql", + "python" + ], + "unique_id": "macro.dbt_spark.materialization_table_spark" + }, + "macro.dbt_spark.materialization_view_spark": { + "arguments": [], + "created_at": 1689766376.398417, + "depends_on": { + "macros": [ + "macro.dbt.create_or_replace_view" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization view, adapter='spark' -%}\n {{ return(create_or_replace_view()) }}\n{%- endmaterialization %}", + "meta": {}, + "name": "materialization_view_spark", + "original_file_path": "macros/materializations/view.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/view.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt_spark.materialization_view_spark" + }, + "macro.dbt_spark.options_clause": { + "arguments": [], + "created_at": 1689766376.376826, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__options_clause" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro options_clause() -%}\n {{ return(adapter.dispatch('options_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "options_clause", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.options_clause" + }, + "macro.dbt_spark.partition_cols": { + "arguments": [], + "created_at": 1689766376.378858, + "depends_on": { + "macros": [ + "macro.dbt_spark.spark__partition_cols" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro partition_cols(label, required=false) %}\n {{ return(adapter.dispatch('partition_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "partition_cols", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.partition_cols" + }, + "macro.dbt_spark.persist_constraints": { + "arguments": [], + "created_at": 1689766376.3827589, + "depends_on": { + "macros": [ + "macro.dbt_databricks.databricks__persist_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", + "meta": {}, + "name": "persist_constraints", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.persist_constraints" + }, + "macro.dbt_spark.py_script_comment": { + "arguments": [], + "created_at": 1689766376.403075, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%macro py_script_comment()%}\n# how to execute python model in notebook\n# dbt = dbtObj(spark.table)\n# df = model(dbt, spark)\n{%endmacro%}", + "meta": {}, + "name": "py_script_comment", + "original_file_path": "macros/materializations/table.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/table.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.py_script_comment" + }, + "macro.dbt_spark.py_write_table": { + "arguments": [], + "created_at": 1689766376.402996, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\n# make sure pyspark exists in the namespace, for 7.3.x-scala2.12 it does not exist\nimport pyspark\n# make sure pandas exists before using it\ntry:\n import pandas\n pandas_available = True\nexcept ImportError:\n pandas_available = False\n\n# make sure pyspark.pandas exists before using it\ntry:\n import pyspark.pandas\n pyspark_pandas_api_available = True\nexcept ImportError:\n pyspark_pandas_api_available = False\n\n# make sure databricks.koalas exists before using it\ntry:\n import databricks.koalas\n koalas_available = True\nexcept ImportError:\n koalas_available = False\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\nif pyspark_pandas_api_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = pyspark.pandas.frame.DataFrame(df)\nelif koalas_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\ndf.write.mode(\"overwrite\").format(\"delta\").option(\"overwriteSchema\", \"true\").saveAsTable(\"{{ target_relation }}\")\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "py_write_table", + "original_file_path": "macros/materializations/table.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/table.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.py_write_table" + }, + "macro.dbt_spark.spark__alter_column_comment": { + "arguments": [], + "created_at": 1689766376.3901749, + "depends_on": { + "macros": [ + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__alter_column_comment(relation, column_dict) %}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'hudi', 'iceberg'] %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set comment_query %}\n {% if relation.is_iceberg %}\n alter table {{ relation }} alter column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% else %}\n alter table {{ relation }} change column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% endif %}\n {% endset %}\n {% do run_query(comment_query) %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "spark__alter_column_comment", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__alter_column_comment" + }, + "macro.dbt_spark.spark__alter_column_set_constraints": { + "arguments": [], + "created_at": 1689766376.3850439, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__alter_column_set_constraints(relation, column_dict) %}\n {% for column_name in column_dict %}\n {% set constraints = column_dict[column_name]['constraints'] %}\n {% for constraint in constraints %}\n {% if constraint.type != 'not_null' %}\n {{ exceptions.warn('Invalid constraint for column ' ~ column_name ~ '. Only `not_null` is supported.') }}\n {% else %}\n {% set quoted_name = adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name %}\n {% call statement() %}\n alter table {{ relation }} change column {{ quoted_name }} set not null {{ constraint.expression or \"\" }};\n {% endcall %}\n {% endif %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "spark__alter_column_set_constraints", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__alter_column_set_constraints" + }, + "macro.dbt_spark.spark__alter_column_type": { + "arguments": [], + "created_at": 1689766376.390744, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__alter_column_type(relation, column_name, new_column_type) -%}\n {% call statement('alter_column_type') %}\n alter table {{ relation }} alter column {{ column_name }} type {{ new_column_type }};\n {% endcall %}\n{% endmacro %}", + "meta": {}, + "name": "spark__alter_column_type", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__alter_column_type" + }, + "macro.dbt_spark.spark__alter_relation_add_remove_columns": { + "arguments": [], + "created_at": 1689766376.391808, + "depends_on": { + "macros": [ + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if remove_columns %}\n {% if relation.is_delta %}\n {% set platform_name = 'Delta Lake' %}\n {% elif relation.is_iceberg %}\n {% set platform_name = 'Iceberg' %}\n {% else %}\n {% set platform_name = 'Apache Spark' %}\n {% endif %}\n {{ exceptions.raise_compiler_error(platform_name + ' does not support dropping columns from tables') }}\n {% endif %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% if add_columns %} add columns {% endif %}\n {% for column in add_columns %}\n {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", + "meta": {}, + "name": "spark__alter_relation_add_remove_columns", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__alter_relation_add_remove_columns" + }, + "macro.dbt_spark.spark__alter_table_add_constraints": { + "arguments": [], + "created_at": 1689766376.384075, + "depends_on": { + "macros": [ + "macro.dbt.is_incremental", + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__alter_table_add_constraints(relation, column_dict) %}\n\n {% for column_name in column_dict %}\n {% set constraints = column_dict[column_name]['constraints'] %}\n {% for constraint in constraints %}\n {% if constraint.type == 'check' and not is_incremental() %}\n {%- set constraint_hash = local_md5(column_name ~ \";\" ~ constraint.expression ~ \";\" ~ loop.index) -%}\n {% call statement() %}\n alter table {{ relation }} add constraint {{ constraint_hash }} check {{ constraint.expression }};\n {% endcall %}\n {% endif %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "spark__alter_table_add_constraints", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__alter_table_add_constraints" + }, + "macro.dbt_spark.spark__any_value": { + "arguments": [], + "created_at": 1689766376.443329, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__any_value(expression) -%}\n {#-- return any value (non-deterministic) --#}\n first({{ expression }})\n\n{%- endmacro %}", + "meta": {}, + "name": "spark__any_value", + "original_file_path": "macros/utils/any_value.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/utils/any_value.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__any_value" + }, + "macro.dbt_spark.spark__array_append": { + "arguments": [], + "created_at": 1689766376.444648, + "depends_on": { + "macros": [ + "macro.dbt.array_concat", + "macro.dbt.array_construct" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__array_append(array, new_element) -%}\n {{ array_concat(array, array_construct([new_element])) }}\n{%- endmacro %}", + "meta": {}, + "name": "spark__array_append", + "original_file_path": "macros/utils/array_append.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/utils/array_append.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__array_append" + }, + "macro.dbt_spark.spark__array_concat": { + "arguments": [], + "created_at": 1689766376.4435031, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__array_concat(array_1, array_2) -%}\n concat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", + "meta": {}, + "name": "spark__array_concat", + "original_file_path": "macros/utils/array_concat.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/utils/array_concat.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__array_concat" + }, + "macro.dbt_spark.spark__array_construct": { + "arguments": [], + "created_at": 1689766376.444437, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__array_construct(inputs, data_type) -%}\n array( {{ inputs|join(' , ') }} )\n{%- endmacro %}", + "meta": {}, + "name": "spark__array_construct", + "original_file_path": "macros/utils/array_construct.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/utils/array_construct.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__array_construct" + }, + "macro.dbt_spark.spark__assert_not_null": { + "arguments": [], + "created_at": 1689766376.431247, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__assert_not_null(function, arg) %}\n\n coalesce({{function}}({{arg}}), nvl2({{function}}({{arg}}), assert_true({{function}}({{arg}}) is not null), null))\n\n{% endmacro %}", + "meta": {}, + "name": "spark__assert_not_null", + "original_file_path": "macros/utils/assert_not_null.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/utils/assert_not_null.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__assert_not_null" + }, + "macro.dbt_spark.spark__bool_or": { + "arguments": [], + "created_at": 1689766376.443661, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__bool_or(expression) -%}\n\n max({{ expression }})\n\n{%- endmacro %}", + "meta": {}, + "name": "spark__bool_or", + "original_file_path": "macros/utils/bool_or.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/utils/bool_or.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__bool_or" + }, + "macro.dbt_spark.spark__call_dcl_statements": { + "arguments": [], + "created_at": 1689766376.3933709, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__call_dcl_statements(dcl_statement_list) %}\n {% for dcl_statement in dcl_statement_list %}\n {% call statement('grant_or_revoke') %}\n {{ dcl_statement }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "spark__call_dcl_statements", + "original_file_path": "macros/apply_grants.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__call_dcl_statements" + }, + "macro.dbt_spark.spark__clustered_cols": { + "arguments": [], + "created_at": 1689766376.380577, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__clustered_cols(label, required=false) %}\n {%- set cols = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n {%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n {%- if (cols is not none) and (buckets is not none) %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n ) into {{ buckets }} buckets\n {%- endif %}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "spark__clustered_cols", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__clustered_cols" + }, + "macro.dbt_spark.spark__comment_clause": { + "arguments": [], + "created_at": 1689766376.378633, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__comment_clause() %}\n {%- set raw_persist_docs = config.get('persist_docs', {}) -%}\n\n {%- if raw_persist_docs is mapping -%}\n {%- set raw_relation = raw_persist_docs.get('relation', false) -%}\n {%- if raw_relation -%}\n comment '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {% endif %}\n {%- elif raw_persist_docs -%}\n {{ exceptions.raise_compiler_error(\"Invalid value provided for 'persist_docs'. Expected dict but got value: \" ~ raw_persist_docs) }}\n {% endif %}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "spark__comment_clause", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__comment_clause" + }, + "macro.dbt_spark.spark__concat": { + "arguments": [], + "created_at": 1689766376.430696, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__concat(fields) -%}\n concat({{ fields|join(', ') }})\n{%- endmacro %}", + "meta": {}, + "name": "spark__concat", + "original_file_path": "macros/utils/concat.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/utils/concat.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__concat" + }, + "macro.dbt_spark.spark__copy_grants": { + "arguments": [], + "created_at": 1689766376.392577, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__copy_grants() %}\n\n {% if config.materialized == 'view' %}\n {#-- Spark views don't copy grants when they're replaced --#}\n {{ return(False) }}\n\n {% else %}\n {#-- This depends on how we're replacing the table, which depends on its file format\n -- Just play it safe by assuming that grants have been copied over, and need to be checked / possibly revoked\n -- We can make this more efficient in the future\n #}\n {{ return(True) }}\n\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "spark__copy_grants", + "original_file_path": "macros/apply_grants.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__copy_grants" + }, + "macro.dbt_spark.spark__create_columns": { + "arguments": [], + "created_at": 1689766376.414269, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__create_columns(relation, columns) %}\n {% if columns|length > 0 %}\n {% call statement() %}\n alter table {{ relation }} add columns (\n {% for column in columns %}\n `{{ column.name }}` {{ column.data_type }} {{- ',' if not loop.last -}}\n {% endfor %}\n );\n {% endcall %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "spark__create_columns", + "original_file_path": "macros/materializations/snapshot.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/snapshot.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__create_columns" + }, + "macro.dbt_spark.spark__create_csv_table": { + "arguments": [], + "created_at": 1689766376.39822, + "depends_on": { + "macros": [ + "macro.dbt_spark.file_format_clause", + "macro.dbt_spark.partition_cols", + "macro.dbt_spark.clustered_cols", + "macro.dbt_spark.location_clause", + "macro.dbt_spark.comment_clause", + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", + "meta": {}, + "name": "spark__create_csv_table", + "original_file_path": "macros/materializations/seed.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/seed.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__create_csv_table" + }, + "macro.dbt_spark.spark__create_schema": { + "arguments": [], + "created_at": 1689766376.38555, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{relation}}\n {% endcall %}\n{% endmacro %}", + "meta": {}, + "name": "spark__create_schema", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__create_schema" + }, + "macro.dbt_spark.spark__create_table_as": { + "arguments": [], + "created_at": 1689766376.3825228, + "depends_on": { + "macros": [ + "macro.dbt_spark.create_temporary_view", + "macro.dbt.get_assert_columns_equivalent", + "macro.dbt.get_select_subquery", + "macro.dbt_spark.file_format_clause", + "macro.dbt_spark.options_clause", + "macro.dbt_spark.partition_cols", + "macro.dbt_spark.clustered_cols", + "macro.dbt_spark.location_clause", + "macro.dbt_spark.comment_clause", + "macro.dbt_spark.py_write_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro spark__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'iceberg'] %}\n create or replace table {{ relation }}\n {% else %}\n create table {{ relation }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause() }}\n {{ options_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "spark__create_table_as", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__create_table_as" + }, + "macro.dbt_spark.spark__create_temporary_view": { + "arguments": [], + "created_at": 1689766376.381353, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__create_temporary_view(relation, compiled_code) -%}\n create temporary view {{ relation }} as\n {{ compiled_code }}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "spark__create_temporary_view", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__create_temporary_view" + }, + "macro.dbt_spark.spark__create_view_as": { + "arguments": [], + "created_at": 1689766376.3853812, + "depends_on": { + "macros": [ + "macro.dbt_spark.comment_clause", + "macro.dbt.get_assert_columns_equivalent" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__create_view_as(relation, sql) -%}\n create or replace view {{ relation }}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as\n {{ sql }}\n{% endmacro %}", + "meta": {}, + "name": "spark__create_view_as", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__create_view_as" + }, + "macro.dbt_spark.spark__current_timestamp": { + "arguments": [], + "created_at": 1689766376.4342709, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__current_timestamp() -%}\n current_timestamp()\n{%- endmacro %}", + "meta": {}, + "name": "spark__current_timestamp", + "original_file_path": "macros/utils/timestamps.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/utils/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__current_timestamp" + }, + "macro.dbt_spark.spark__dateadd": { + "arguments": [], + "created_at": 1689766376.434155, + "depends_on": { + "macros": [ + "macro.dbt_spark.assert_not_null" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {%- set clock_component -%}\n {# make sure the dates + timestamps are real, otherwise raise an error asap #}\n to_unix_timestamp({{ assert_not_null('to_timestamp', from_date_or_timestamp) }})\n - to_unix_timestamp({{ assert_not_null('date', from_date_or_timestamp) }})\n {%- endset -%}\n\n {%- if datepart in ['day', 'week'] -%}\n\n {%- set multiplier = 7 if datepart == 'week' else 1 -%}\n\n to_timestamp(\n to_unix_timestamp(\n date_add(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ['month', 'quarter', 'year'] -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'month' -%} 1\n {%- elif datepart == 'quarter' -%} 3\n {%- elif datepart == 'year' -%} 12\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n to_unix_timestamp(\n add_months(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n {{ assert_not_null('to_unix_timestamp', from_date_or_timestamp) }}\n + cast({{interval}} * {{multiplier}} as int)\n )\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro dateadd not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", + "meta": {}, + "name": "spark__dateadd", + "original_file_path": "macros/utils/dateadd.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/utils/dateadd.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__dateadd" + }, + "macro.dbt_spark.spark__datediff": { + "arguments": [], + "created_at": 1689766376.443135, + "depends_on": { + "macros": [ + "macro.dbt_spark.assert_not_null" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__datediff(first_date, second_date, datepart) %}\n\n {%- if datepart in ['day', 'week', 'month', 'quarter', 'year'] -%}\n\n {# make sure the dates are real, otherwise raise an error asap #}\n {% set first_date = assert_not_null('date', first_date) %}\n {% set second_date = assert_not_null('date', second_date) %}\n\n {%- endif -%}\n\n {%- if datepart == 'day' -%}\n\n datediff({{second_date}}, {{first_date}})\n\n {%- elif datepart == 'week' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(datediff({{second_date}}, {{first_date}})/7)\n else ceil(datediff({{second_date}}, {{first_date}})/7)\n end\n\n -- did we cross a week boundary (Sunday)?\n + case\n when {{first_date}} < {{second_date}} and dayofweek({{second_date}}) < dayofweek({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofweek({{second_date}}) > dayofweek({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'month' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}})))\n else ceil(months_between(date({{second_date}}), date({{first_date}})))\n end\n\n -- did we cross a month boundary?\n + case\n when {{first_date}} < {{second_date}} and dayofmonth({{second_date}}) < dayofmonth({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofmonth({{second_date}}) > dayofmonth({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'quarter' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}}))/3)\n else ceil(months_between(date({{second_date}}), date({{first_date}}))/3)\n end\n\n -- did we cross a quarter boundary?\n + case\n when {{first_date}} < {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n < (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then 1\n when {{first_date}} > {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n > (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then -1\n else 0 end\n\n {%- elif datepart == 'year' -%}\n\n year({{second_date}}) - year({{first_date}})\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set divisor -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n case when {{first_date}} < {{second_date}}\n then ceil((\n {# make sure the timestamps are real, otherwise raise an error asap #}\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n else floor((\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n end\n\n {% if datepart == 'millisecond' %}\n + cast(date_format({{second_date}}, 'SSS') as int)\n - cast(date_format({{first_date}}, 'SSS') as int)\n {% endif %}\n\n {% if datepart == 'microsecond' %}\n {% set capture_str = '[0-9]{4}-[0-9]{2}-[0-9]{2}.[0-9]{2}:[0-9]{2}:[0-9]{2}.([0-9]{6})' %}\n -- Spark doesn't really support microseconds, so this is a massive hack!\n -- It will only work if the timestamp-string is of the format\n -- 'yyyy-MM-dd-HH mm.ss.SSSSSS'\n + cast(regexp_extract({{second_date}}, '{{capture_str}}', 1) as int)\n - cast(regexp_extract({{first_date}}, '{{capture_str}}', 1) as int)\n {% endif %}\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro datediff not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", + "meta": {}, + "name": "spark__datediff", + "original_file_path": "macros/utils/datediff.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/utils/datediff.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__datediff" + }, + "macro.dbt_spark.spark__drop_relation": { + "arguments": [], + "created_at": 1689766376.388672, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }}\n {%- endcall %}\n{% endmacro %}", + "meta": {}, + "name": "spark__drop_relation", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__drop_relation" + }, + "macro.dbt_spark.spark__drop_schema": { + "arguments": [], + "created_at": 1689766376.385717, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation }} cascade\n {%- endcall -%}\n{% endmacro %}", + "meta": {}, + "name": "spark__drop_schema", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__drop_schema" + }, + "macro.dbt_spark.spark__escape_single_quotes": { + "arguments": [], + "created_at": 1689766376.434459, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"\\\\'\") }}\n{%- endmacro %}", + "meta": {}, + "name": "spark__escape_single_quotes", + "original_file_path": "macros/utils/escape_single_quotes.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/utils/escape_single_quotes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__escape_single_quotes" + }, + "macro.dbt_spark.spark__file_format_clause": { + "arguments": [], + "created_at": 1689766376.376118, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__file_format_clause() %}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n {%- if file_format is not none %}\n using {{ file_format }}\n {%- endif %}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "spark__file_format_clause", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__file_format_clause" + }, + "macro.dbt_spark.spark__generate_database_name": { + "arguments": [], + "created_at": 1689766376.3888378, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(None) %}\n{%- endmacro %}", + "meta": {}, + "name": "spark__generate_database_name", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__generate_database_name" + }, + "macro.dbt_spark.spark__get_binding_char": { + "arguments": [], + "created_at": 1689766376.3952699, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__get_binding_char() %}\n {{ return('?' if target.method == 'odbc' else '%s') }}\n{% endmacro %}", + "meta": {}, + "name": "spark__get_binding_char", + "original_file_path": "macros/materializations/seed.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/seed.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__get_binding_char" + }, + "macro.dbt_spark.spark__get_columns_in_relation": { + "arguments": [], + "created_at": 1689766376.386521, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n describe extended {{ relation.include(schema=(schema is not none)) }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation').table) %}\n{% endmacro %}", + "meta": {}, + "name": "spark__get_columns_in_relation", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__get_columns_in_relation" + }, + "macro.dbt_spark.spark__get_columns_in_relation_raw": { + "arguments": [], + "created_at": 1689766376.386165, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__get_columns_in_relation_raw(relation) -%}\n {% call statement('get_columns_in_relation_raw', fetch_result=True) %}\n describe extended {{ relation }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation_raw').table) %}\n{% endmacro %}", + "meta": {}, + "name": "spark__get_columns_in_relation_raw", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__get_columns_in_relation_raw" + }, + "macro.dbt_spark.spark__get_grant_sql": { + "arguments": [], + "created_at": 1689766376.392803, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n\n{%- macro spark__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", + "meta": {}, + "name": "spark__get_grant_sql", + "original_file_path": "macros/apply_grants.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__get_grant_sql" + }, + "macro.dbt_spark.spark__get_merge_sql": { + "arguments": [], + "created_at": 1689766376.4260302, + "depends_on": { + "macros": [ + "macro.dbt.get_merge_update_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n when matched then update set\n {% if update_columns -%}{%- for column_name in update_columns %}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {%- else %} * {% endif %}\n\n when not matched then insert *\n{% endmacro %}", + "meta": {}, + "name": "spark__get_merge_sql", + "original_file_path": "macros/materializations/incremental/strategies.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__get_merge_sql" + }, + "macro.dbt_spark.spark__get_merge_update_columns": { + "arguments": [], + "created_at": 1689766376.419403, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = None -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", + "meta": {}, + "name": "spark__get_merge_update_columns", + "original_file_path": "macros/materializations/incremental/column_helpers.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__get_merge_update_columns" + }, + "macro.dbt_spark.spark__get_revoke_sql": { + "arguments": [], + "created_at": 1689766376.393013, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n\n{%- macro spark__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", + "meta": {}, + "name": "spark__get_revoke_sql", + "original_file_path": "macros/apply_grants.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__get_revoke_sql" + }, + "macro.dbt_spark.spark__list_relations_without_caching": { + "arguments": [], + "created_at": 1689766376.386868, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__list_relations_without_caching(relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n show table extended in {{ relation }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching').table) %}\n{% endmacro %}", + "meta": {}, + "name": "spark__list_relations_without_caching", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__list_relations_without_caching" + }, + "macro.dbt_spark.spark__list_schemas": { + "arguments": [], + "created_at": 1689766376.387706, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__list_schemas(database) -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n show databases\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", + "meta": {}, + "name": "spark__list_schemas", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__list_schemas" + }, + "macro.dbt_spark.spark__listagg": { + "arguments": [], + "created_at": 1689766376.435216, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if order_by_clause %}\n {{ exceptions.warn(\"order_by_clause is not supported for listagg on Spark/Databricks\") }}\n {% endif %}\n\n {% set collect_list %} collect_list({{ measure }}) {% endset %}\n\n {% set limited %} slice({{ collect_list }}, 1, {{ limit_num }}) {% endset %}\n\n {% set collected = limited if limit_num else collect_list %}\n\n {% set final %} array_join({{ collected }}, {{ delimiter_text }}) {% endset %}\n\n {% do return(final) %}\n\n{%- endmacro %}", + "meta": {}, + "name": "spark__listagg", + "original_file_path": "macros/utils/listagg.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/utils/listagg.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__listagg" + }, + "macro.dbt_spark.spark__load_csv_rows": { + "arguments": [], + "created_at": 1689766376.397096, + "depends_on": { + "macros": [ + "macro.dbt.get_batch_size", + "macro.dbt.get_binding_char" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", + "meta": {}, + "name": "spark__load_csv_rows", + "original_file_path": "macros/materializations/seed.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/seed.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__load_csv_rows" + }, + "macro.dbt_spark.spark__location_clause": { + "arguments": [], + "created_at": 1689766376.376653, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__location_clause() %}\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n location '{{ location_root }}/{{ identifier }}'\n {%- endif %}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "spark__location_clause", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__location_clause" + }, + "macro.dbt_spark.spark__make_temp_relation": { + "arguments": [], + "created_at": 1689766376.390503, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(path = {\n \"identifier\": tmp_identifier\n }) -%}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", + "meta": {}, + "name": "spark__make_temp_relation", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__make_temp_relation" + }, + "macro.dbt_spark.spark__options_clause": { + "arguments": [], + "created_at": 1689766376.377955, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__options_clause() -%}\n {%- set options = config.get('options') -%}\n {%- if config.get('file_format') == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "spark__options_clause", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__options_clause" + }, + "macro.dbt_spark.spark__partition_cols": { + "arguments": [], + "created_at": 1689766376.3794, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__partition_cols(label, required=false) %}\n {%- set cols = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "spark__partition_cols", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__partition_cols" + }, + "macro.dbt_spark.spark__persist_constraints": { + "arguments": [], + "created_at": 1689766376.3832, + "depends_on": { + "macros": [ + "macro.dbt_spark.alter_table_add_constraints", + "macro.dbt_spark.alter_column_set_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config.enforced and config.get('file_format', 'delta') == 'delta' %}\n {% do alter_table_add_constraints(relation, model.columns) %}\n {% do alter_column_set_constraints(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "spark__persist_constraints", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__persist_constraints" + }, + "macro.dbt_spark.spark__persist_docs": { + "arguments": [], + "created_at": 1689766376.389123, + "depends_on": { + "macros": [ + "macro.dbt.alter_column_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do alter_column_comment(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "spark__persist_docs", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__persist_docs" + }, + "macro.dbt_spark.spark__post_snapshot": { + "arguments": [], + "created_at": 1689766376.413843, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__post_snapshot(staging_relation) %}\n {% do adapter.drop_relation(staging_relation) %}\n{% endmacro %}", + "meta": {}, + "name": "spark__post_snapshot", + "original_file_path": "macros/materializations/snapshot.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/snapshot.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__post_snapshot" + }, + "macro.dbt_spark.spark__rename_relation": { + "arguments": [], + "created_at": 1689766376.388437, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__rename_relation(from_relation, to_relation) -%}\n {% call statement('rename_relation') -%}\n {% if not from_relation.type %}\n {% do exceptions.raise_database_error(\"Cannot rename a relation with a blank type: \" ~ from_relation.identifier) %}\n {% elif from_relation.type in ('table') %}\n alter table {{ from_relation }} rename to {{ to_relation }}\n {% elif from_relation.type == 'view' %}\n alter view {{ from_relation }} rename to {{ to_relation }}\n {% else %}\n {% do exceptions.raise_database_error(\"Unknown type '\" ~ from_relation.type ~ \"' for relation: \" ~ from_relation.identifier) %}\n {% endif %}\n {%- endcall %}\n{% endmacro %}", + "meta": {}, + "name": "spark__rename_relation", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__rename_relation" + }, + "macro.dbt_spark.spark__reset_csv_table": { + "arguments": [], + "created_at": 1689766376.395597, + "depends_on": { + "macros": [ + "macro.dbt.create_csv_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {{ adapter.drop_relation(old_relation) }}\n {% endif %}\n {% set sql = create_csv_table(model, agate_table) %}\n {{ return(sql) }}\n{% endmacro %}", + "meta": {}, + "name": "spark__reset_csv_table", + "original_file_path": "macros/materializations/seed.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/seed.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__reset_csv_table" + }, + "macro.dbt_spark.spark__snapshot_hash_arguments": { + "arguments": [], + "created_at": 1689766376.412252, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as string ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", + "meta": {}, + "name": "spark__snapshot_hash_arguments", + "original_file_path": "macros/materializations/snapshot.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/snapshot.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__snapshot_hash_arguments" + }, + "macro.dbt_spark.spark__snapshot_merge_sql": { + "arguments": [], + "created_at": 1689766376.412747, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__snapshot_merge_sql(target, source, insert_cols) -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n {% if target.is_iceberg %}\n {# create view only supports a name (no catalog, or schema) #}\n using {{ source.identifier }} as DBT_INTERNAL_SOURCE\n {% else %}\n using {{ source }} as DBT_INTERNAL_SOURCE\n {% endif %}\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert *\n ;\n{% endmacro %}", + "meta": {}, + "name": "spark__snapshot_merge_sql", + "original_file_path": "macros/materializations/snapshot.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/snapshot.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__snapshot_merge_sql" + }, + "macro.dbt_spark.spark__snapshot_string_as_time": { + "arguments": [], + "created_at": 1689766376.4124532, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"to_timestamp('\" ~ timestamp ~ \"')\" -%}\n {{ return(result) }}\n{%- endmacro %}", + "meta": {}, + "name": "spark__snapshot_string_as_time", + "original_file_path": "macros/materializations/snapshot.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/snapshot.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__snapshot_string_as_time" + }, + "macro.dbt_spark.spark__split_part": { + "arguments": [], + "created_at": 1689766376.444246, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark__split_part(string_text, delimiter_text, part_number) %}\n\n {% set delimiter_expr %}\n\n -- escape if starts with a special character\n case when regexp_extract({{ delimiter_text }}, '([^A-Za-z0-9])(.*)', 1) != '_'\n then concat('\\\\', {{ delimiter_text }})\n else {{ delimiter_text }} end\n\n {% endset %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[({{ part_number - 1 }})]\n\n {% endset %}\n\n {{ return(split_part_expr) }}\n\n{% endmacro %}", + "meta": {}, + "name": "spark__split_part", + "original_file_path": "macros/utils/split_part.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/utils/split_part.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__split_part" + }, + "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement": { + "arguments": [], + "created_at": 1689766376.3931332, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n\n{%- macro spark__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(False) }}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "spark__support_multiple_grantees_per_dcl_statement", + "original_file_path": "macros/apply_grants.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement" + }, + "macro.dbt_spark.spark_build_snapshot_staging_table": { + "arguments": [], + "created_at": 1689766376.4136682, + "depends_on": { + "macros": [ + "macro.dbt.snapshot_staging_table", + "macro.dbt.statement", + "macro.dbt.create_view_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_identifier = target_relation.identifier ~ '__dbt_tmp' %}\n\n {% if target_relation.is_iceberg %}\n {# iceberg catalog does not support create view, but regular spark does. We removed the catalog and schema #}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=none,\n database=none,\n type='view') -%}\n {% else %}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=target_relation.schema,\n database=none,\n type='view') -%}\n {% endif %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_view_as(tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", + "meta": {}, + "name": "spark_build_snapshot_staging_table", + "original_file_path": "macros/materializations/snapshot.sql", + "package_name": "dbt_spark", + "patch_path": null, + "path": "macros/materializations/snapshot.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_spark.spark_build_snapshot_staging_table" + }, + "macro.jaffle_shop.drop_table": { + "arguments": [], + "created_at": 1689766376.242068, + "depends_on": { + "macros": [ + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro drop_table(table_name) -%}\n {%- set drop_query -%}\n DROP TABLE IF EXISTS {{ target.schema }}.{{ table_name }} CASCADE\n {%- endset -%}\n {% do run_query(drop_query) %}\n{%- endmacro -%}", + "meta": {}, + "name": "drop_table", + "original_file_path": "macros/drop_table.sql", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "macros/drop_table.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.jaffle_shop.drop_table" + } + }, + "metadata": { + "adapter_type": "databricks", + "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v9.json", + "dbt_version": "1.5.0", + "env": {}, + "generated_at": "2023-07-19T22:36:01.468587Z", + "invocation_id": "dcaa6d07-b65d-4943-8f54-f5496635211d", + "project_id": "06e5b98c2db46f8a72cc4f66410e9b3b", + "send_anonymous_usage_stats": true, + "user_id": "5efd653d-504b-443e-8880-535eea840161" + }, + "metrics": {}, + "nodes": { + "model.jaffle_shop.customers": { + "access": "protected", + "alias": "customers", + "build_path": null, + "checksum": { + "checksum": "95ddc646d940ee1e702612f3d69650d1794a2c20114085ac461377cbfd5ef21b", + "name": "sha256" + }, + "columns": { + "customer_id": { + "constraints": [], + "data_type": null, + "description": "This is a unique identifier for a customer", + "meta": {}, + "name": "customer_id", + "quote": null, + "tags": [] + }, + "first_name": { + "constraints": [], + "data_type": null, + "description": "Customer's first name. PII.", + "meta": {}, + "name": "first_name", + "quote": null, + "tags": [] + }, + "first_order": { + "constraints": [], + "data_type": null, + "description": "Date (UTC) of a customer's first order", + "meta": {}, + "name": "first_order", + "quote": null, + "tags": [] + }, + "last_name": { + "constraints": [], + "data_type": null, + "description": "Customer's last name. PII.", + "meta": {}, + "name": "last_name", + "quote": null, + "tags": [] + }, + "most_recent_order": { + "constraints": [], + "data_type": null, + "description": "Date (UTC) of a customer's most recent order", + "meta": {}, + "name": "most_recent_order", + "quote": null, + "tags": [] + }, + "number_of_orders": { + "constraints": [], + "data_type": null, + "description": "Count of the number of orders a customer has placed", + "meta": {}, + "name": "number_of_orders", + "quote": null, + "tags": [] + }, + "total_order_amount": { + "constraints": [], + "data_type": null, + "description": "Total value (AUD) of a customer's orders", + "meta": {}, + "name": "total_order_amount", + "quote": null, + "tags": [] + } + }, + "compiled_path": null, + "config": { + "alias": null, + "cluster_id": "0403-094356-wab883hn", + "column_types": {}, + "contract": { + "enforced": false + }, + "create_notebook": false, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "table", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "submission_method": "all_purpose_cluster", + "tags": [], + "unique_key": null + }, + "constraints": [], + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.852364, + "database": null, + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "model.jaffle_shop.stg_customers", + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments" + ] + }, + "description": "This table has basic information about a customer, as well as some derived facts based on a customer's orders", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "customers" + ], + "group": null, + "language": "python", + "latest_version": null, + "meta": {}, + "metrics": [], + "name": "customers", + "original_file_path": "models/customers.py", + "package_name": "jaffle_shop", + "patch_path": "jaffle_shop://models/schema.yml", + "path": "customers.py", + "raw_code": "import pyspark.sql.functions as F\n\n\ndef model(dbt, session):\n stg_customers_df = dbt.ref(\"stg_customers\")\n stg_orders_df = dbt.ref(\"stg_orders\")\n stg_payments_df = dbt.ref(\"stg_payments\")\n\n customer_orders_df = stg_orders_df.groupby(\"customer_id\").agg(\n F.min(F.col(\"order_date\")).alias(\"first_order\"),\n F.max(F.col(\"order_date\")).alias(\"most_recent_order\"),\n F.count(F.col(\"order_id\")).alias(\"number_of_orders\"),\n )\n\n customer_payments_df = (\n stg_payments_df.join(stg_orders_df, stg_payments_df.order_id == stg_orders_df.order_id, \"left\")\n .groupby(stg_orders_df.customer_id)\n .agg(F.sum(F.col(\"amount\")).alias(\"total_amount\"))\n )\n\n final_df = (\n stg_customers_df.alias(\"customers\")\n .join(\n customer_orders_df.alias(\"customer_orders\"),\n F.col(\"customers.customer_id\") == F.col(\"customer_orders.customer_id\"),\n \"left\",\n )\n .join(\n customer_payments_df.alias(\"customer_payments\"),\n F.col(\"customers.customer_id\") == F.col(\"customer_payments.customer_id\"),\n \"left\",\n )\n .select(\n F.col(\"customers.customer_id\").alias(\"customer_id\"),\n F.col(\"customers.first_name\").alias(\"first_name\"),\n F.col(\"customers.last_name\").alias(\"last_name\"),\n F.col(\"customer_orders.first_order\").alias(\"first_order\"),\n F.col(\"customer_orders.most_recent_order\").alias(\"most_recent_order\"),\n F.col(\"customer_orders.number_of_orders\").alias(\"number_of_orders\"),\n F.col(\"customer_payments.total_amount\").alias(\"customer_lifetime_value\"),\n )\n )\n\n return final_df", + "refs": [ + { + "name": "stg_customers", + "package": null, + "version": null + }, + { + "name": "stg_orders", + "package": null, + "version": null + }, + { + "name": "stg_payments", + "package": null, + "version": null + } + ], + "relation_name": "`cosmos`.`customers`", + "resource_type": "model", + "schema": "cosmos", + "sources": [], + "tags": [], + "unique_id": "model.jaffle_shop.customers", + "unrendered_config": { + "cluster_id": "0403-094356-wab883hn", + "create_notebook": false, + "materialized": "table", + "submission_method": "all_purpose_cluster" + }, + "version": null + }, + "model.jaffle_shop.orders": { + "access": "protected", + "alias": "orders", + "build_path": null, + "checksum": { + "checksum": "b6553d2a3e7a77b00b0afdb9d53f7ac4de548c939041109c147765c9dda91b1e", + "name": "sha256" + }, + "columns": { + "amount": { + "constraints": [], + "data_type": null, + "description": "Total amount (AUD) of the order", + "meta": {}, + "name": "amount", + "quote": null, + "tags": [] + }, + "bank_transfer_amount": { + "constraints": [], + "data_type": null, + "description": "Amount of the order (AUD) paid for by bank transfer", + "meta": {}, + "name": "bank_transfer_amount", + "quote": null, + "tags": [] + }, + "coupon_amount": { + "constraints": [], + "data_type": null, + "description": "Amount of the order (AUD) paid for by coupon", + "meta": {}, + "name": "coupon_amount", + "quote": null, + "tags": [] + }, + "credit_card_amount": { + "constraints": [], + "data_type": null, + "description": "Amount of the order (AUD) paid for by credit card", + "meta": {}, + "name": "credit_card_amount", + "quote": null, + "tags": [] + }, + "customer_id": { + "constraints": [], + "data_type": null, + "description": "Foreign key to the customers table", + "meta": {}, + "name": "customer_id", + "quote": null, + "tags": [] + }, + "gift_card_amount": { + "constraints": [], + "data_type": null, + "description": "Amount of the order (AUD) paid for by gift card", + "meta": {}, + "name": "gift_card_amount", + "quote": null, + "tags": [] + }, + "order_date": { + "constraints": [], + "data_type": null, + "description": "Date (UTC) that the order was placed", + "meta": {}, + "name": "order_date", + "quote": null, + "tags": [] + }, + "order_id": { + "constraints": [], + "data_type": null, + "description": "This is a unique identifier for an order", + "meta": {}, + "name": "order_id", + "quote": null, + "tags": [] + }, + "status": { + "constraints": [], + "data_type": null, + "description": "Orders can be one of the following statuses:\n\n| status | description |\n|----------------|------------------------------------------------------------------------------------------------------------------------|\n| placed | The order has been placed but has not yet left the warehouse |\n| shipped | The order has ben shipped to the customer and is currently in transit |\n| completed | The order has been received by the customer |\n| return_pending | The customer has indicated that they would like to return the order, but it has not yet been received at the warehouse |\n| returned | The order has been returned by the customer and received at the warehouse |", + "meta": {}, + "name": "status", + "quote": null, + "tags": [] + } + }, + "compiled_path": null, + "config": { + "alias": null, + "cluster_id": "0403-094356-wab883hn", + "column_types": {}, + "contract": { + "enforced": false + }, + "create_notebook": false, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "table", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "submission_method": "all_purpose_cluster", + "tags": [], + "unique_key": null + }, + "constraints": [], + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.8561542, + "database": null, + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments" + ] + }, + "description": "This table has basic information about orders, as well as some derived facts based on payments", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "orders" + ], + "group": null, + "language": "python", + "latest_version": null, + "meta": {}, + "metrics": [], + "name": "orders", + "original_file_path": "models/orders.py", + "package_name": "jaffle_shop", + "patch_path": "jaffle_shop://models/schema.yml", + "path": "orders.py", + "raw_code": "import pyspark.sql.functions as F\n\n\ndef model(dbt, session):\n stg_orders_df = dbt.ref(\"stg_orders\")\n stg_payments_df = dbt.ref(\"stg_payments\")\n\n payment_methods = [\"credit_card\", \"coupon\", \"bank_transfer\", \"gift_card\"]\n\n agg_list = [\n F.sum(F.when(stg_payments_df.payment_method == payment_method, stg_payments_df.amount).otherwise(0)).alias(\n payment_method + \"_amount\"\n )\n for payment_method in payment_methods\n ]\n agg_list.append(F.sum(F.col(\"amount\")).alias(\"total_amount\"))\n\n order_payments_df = stg_payments_df.groupby(\"order_id\").agg(*agg_list)\n\n final_df = stg_orders_df.join(\n order_payments_df, stg_orders_df.order_id == order_payments_df.order_id, \"left\"\n ).select(\n stg_orders_df.order_id.alias(\"order_id\"),\n stg_orders_df.customer_id.alias(\"customer_id\"),\n stg_orders_df.order_date.alias(\"order_date\"),\n stg_orders_df.status.alias(\"status\"),\n *[F.col(payment_method + \"_amount\") for payment_method in payment_methods],\n order_payments_df.total_amount.alias(\"amount\"),\n )\n\n return final_df", + "refs": [ + { + "name": "stg_orders", + "package": null, + "version": null + }, + { + "name": "stg_payments", + "package": null, + "version": null + } + ], + "relation_name": "`cosmos`.`orders`", + "resource_type": "model", + "schema": "cosmos", + "sources": [], + "tags": [], + "unique_id": "model.jaffle_shop.orders", + "unrendered_config": { + "cluster_id": "0403-094356-wab883hn", + "create_notebook": false, + "materialized": "table", + "submission_method": "all_purpose_cluster" + }, + "version": null + }, + "model.jaffle_shop.stg_customers": { + "access": "protected", + "alias": "stg_customers", + "build_path": null, + "checksum": { + "checksum": "80e3223cd54387e11fa16cd0f4cbe15f8ff74dcd9900b93856b9e39416178c9d", + "name": "sha256" + }, + "columns": { + "customer_id": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "customer_id", + "quote": null, + "tags": [] + } + }, + "compiled_path": null, + "config": { + "alias": null, + "column_types": {}, + "contract": { + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "view", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "constraints": [], + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.887611, + "database": null, + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "seed.jaffle_shop.raw_customers" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "staging", + "stg_customers" + ], + "group": null, + "language": "sql", + "latest_version": null, + "meta": {}, + "metrics": [], + "name": "stg_customers", + "original_file_path": "models/staging/stg_customers.sql", + "package_name": "jaffle_shop", + "patch_path": "jaffle_shop://models/staging/schema.yml", + "path": "staging/stg_customers.sql", + "raw_code": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_customers') }}\n\n),\n\nrenamed as (\n\n select\n id as customer_id,\n first_name,\n last_name\n\n from source\n\n)\n\nselect * from renamed", + "refs": [ + { + "name": "raw_customers", + "package": null, + "version": null + } + ], + "relation_name": "`cosmos`.`stg_customers`", + "resource_type": "model", + "schema": "cosmos", + "sources": [], + "tags": [], + "unique_id": "model.jaffle_shop.stg_customers", + "unrendered_config": {}, + "version": null + }, + "model.jaffle_shop.stg_orders": { + "access": "protected", + "alias": "stg_orders", + "build_path": null, + "checksum": { + "checksum": "f4f881cb09d2c4162200fc331d7401df6d1abd4fed492554a7db70dede347108", + "name": "sha256" + }, + "columns": { + "order_id": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "order_id", + "quote": null, + "tags": [] + }, + "status": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "status", + "quote": null, + "tags": [] + } + }, + "compiled_path": null, + "config": { + "alias": null, + "column_types": {}, + "contract": { + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "view", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "constraints": [], + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.888557, + "database": null, + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "seed.jaffle_shop.raw_orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "staging", + "stg_orders" + ], + "group": null, + "language": "sql", + "latest_version": null, + "meta": {}, + "metrics": [], + "name": "stg_orders", + "original_file_path": "models/staging/stg_orders.sql", + "package_name": "jaffle_shop", + "patch_path": "jaffle_shop://models/staging/schema.yml", + "path": "staging/stg_orders.sql", + "raw_code": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_orders') }}\n\n),\n\nrenamed as (\n\n select\n id as order_id,\n user_id as customer_id,\n order_date,\n status\n\n from source\n\n)\n\nselect * from renamed", + "refs": [ + { + "name": "raw_orders", + "package": null, + "version": null + } + ], + "relation_name": "`cosmos`.`stg_orders`", + "resource_type": "model", + "schema": "cosmos", + "sources": [], + "tags": [], + "unique_id": "model.jaffle_shop.stg_orders", + "unrendered_config": {}, + "version": null + }, + "model.jaffle_shop.stg_payments": { + "access": "protected", + "alias": "stg_payments", + "build_path": null, + "checksum": { + "checksum": "30f346f66ef7bca4c8865a471086303720c3daab58870c805b6f45e92d19fd65", + "name": "sha256" + }, + "columns": { + "payment_id": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "payment_id", + "quote": null, + "tags": [] + }, + "payment_method": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "payment_method", + "quote": null, + "tags": [] + } + }, + "compiled_path": null, + "config": { + "alias": null, + "column_types": {}, + "contract": { + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "view", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "constraints": [], + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.889248, + "database": null, + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "seed.jaffle_shop.raw_payments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "staging", + "stg_payments" + ], + "group": null, + "language": "sql", + "latest_version": null, + "meta": {}, + "metrics": [], + "name": "stg_payments", + "original_file_path": "models/staging/stg_payments.sql", + "package_name": "jaffle_shop", + "patch_path": "jaffle_shop://models/staging/schema.yml", + "path": "staging/stg_payments.sql", + "raw_code": "with source as (\n\n {#-\n Normally we would select from the table here, but we are using seeds to load\n our data in this project\n #}\n select * from {{ ref('raw_payments') }}\n\n),\n\nrenamed as (\n\n select\n id as payment_id,\n order_id,\n payment_method,\n\n -- `amount` is currently stored in cents, so we convert it to dollars\n amount / 100 as amount\n\n from source\n\n)\n\nselect * from renamed", + "refs": [ + { + "name": "raw_payments", + "package": null, + "version": null + } + ], + "relation_name": "`cosmos`.`stg_payments`", + "resource_type": "model", + "schema": "cosmos", + "sources": [], + "tags": [], + "unique_id": "model.jaffle_shop.stg_payments", + "unrendered_config": {}, + "version": null + }, + "seed.jaffle_shop.raw_customers": { + "alias": "raw_customers", + "build_path": null, + "checksum": { + "checksum": "357d173dda65a741ad97d6683502286cc2655bb396ab5f4dfad12b8c39bd2a63", + "name": "sha256" + }, + "columns": {}, + "config": { + "alias": null, + "column_types": {}, + "contract": { + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "seed", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quote_columns": null, + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "created_at": 1689766376.830008, + "database": null, + "deferred": false, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "raw_customers" + ], + "group": null, + "meta": {}, + "name": "raw_customers", + "original_file_path": "seeds/raw_customers.csv", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "raw_customers.csv", + "raw_code": "", + "relation_name": "`cosmos`.`raw_customers`", + "resource_type": "seed", + "root_path": "/Users/tati/Code/astronomer-cosmos/dev/dags/dbt/jaffle_shop_python", + "schema": "cosmos", + "tags": [], + "unique_id": "seed.jaffle_shop.raw_customers", + "unrendered_config": {} + }, + "seed.jaffle_shop.raw_orders": { + "alias": "raw_orders", + "build_path": null, + "checksum": { + "checksum": "6228dde8e17b9621f35c13e272ec67d3ff55b55499433f47d303adf2be72c17f", + "name": "sha256" + }, + "columns": {}, + "config": { + "alias": null, + "column_types": {}, + "contract": { + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "seed", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quote_columns": null, + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "created_at": 1689766376.831657, + "database": null, + "deferred": false, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "raw_orders" + ], + "group": null, + "meta": {}, + "name": "raw_orders", + "original_file_path": "seeds/raw_orders.csv", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "raw_orders.csv", + "raw_code": "", + "relation_name": "`cosmos`.`raw_orders`", + "resource_type": "seed", + "root_path": "/Users/tati/Code/astronomer-cosmos/dev/dags/dbt/jaffle_shop_python", + "schema": "cosmos", + "tags": [], + "unique_id": "seed.jaffle_shop.raw_orders", + "unrendered_config": {} + }, + "seed.jaffle_shop.raw_payments": { + "alias": "raw_payments", + "build_path": null, + "checksum": { + "checksum": "6de0626a8db9c1750eefd1b2e17fac4c2a4b9f778eb50532d8b377b90de395e6", + "name": "sha256" + }, + "columns": {}, + "config": { + "alias": null, + "column_types": {}, + "contract": { + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "seed", + "meta": {}, + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quote_columns": null, + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "created_at": 1689766376.833125, + "database": null, + "deferred": false, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "fqn": [ + "jaffle_shop", + "raw_payments" + ], + "group": null, + "meta": {}, + "name": "raw_payments", + "original_file_path": "seeds/raw_payments.csv", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "raw_payments.csv", + "raw_code": "", + "relation_name": "`cosmos`.`raw_payments`", + "resource_type": "seed", + "root_path": "/Users/tati/Code/astronomer-cosmos/dev/dags/dbt/jaffle_shop_python", + "schema": "cosmos", + "tags": [], + "unique_id": "seed.jaffle_shop.raw_payments", + "unrendered_config": {} + }, + "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3": { + "alias": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "status", + "columns": {}, + "compiled_path": null, + "config": { + "alias": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758", + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.876723, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_accepted_values", + "macro.dbt.get_where_subquery" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "accepted_values_orders_status__placed__shipped__completed__return_pending__returned" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "accepted_values_orders_status__placed__shipped__completed__return_pending__returned", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758.sql", + "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758\") }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "status", + "model": "{{ get_where_subquery(ref('orders')) }}", + "values": [ + "placed", + "shipped", + "completed", + "return_pending", + "returned" + ] + }, + "name": "accepted_values", + "namespace": null + }, + "unique_id": "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3", + "unrendered_config": { + "alias": "accepted_values_orders_1ce6ab157c285f7cd2ac656013faf758" + } + }, + "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad": { + "alias": "accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58", + "attached_node": "model.jaffle_shop.stg_orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "status", + "columns": {}, + "compiled_path": null, + "config": { + "alias": "accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58", + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.894539, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_accepted_values", + "macro.dbt.get_where_subquery" + ], + "nodes": [ + "model.jaffle_shop.stg_orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_orders", + "fqn": [ + "jaffle_shop", + "staging", + "accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58.sql", + "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58\") }}", + "refs": [ + { + "name": "stg_orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "status", + "model": "{{ get_where_subquery(ref('stg_orders')) }}", + "values": [ + "placed", + "shipped", + "completed", + "return_pending", + "returned" + ] + }, + "name": "accepted_values", + "namespace": null + }, + "unique_id": "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad", + "unrendered_config": { + "alias": "accepted_values_stg_orders_4f514bf94b77b7ea437830eec4421c58" + } + }, + "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278": { + "alias": "accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef", + "attached_node": "model.jaffle_shop.stg_payments", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "payment_method", + "columns": {}, + "compiled_path": null, + "config": { + "alias": "accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef", + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.901102, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_accepted_values", + "macro.dbt.get_where_subquery" + ], + "nodes": [ + "model.jaffle_shop.stg_payments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_payments", + "fqn": [ + "jaffle_shop", + "staging", + "accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef.sql", + "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef\") }}", + "refs": [ + { + "name": "stg_payments", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "payment_method", + "model": "{{ get_where_subquery(ref('stg_payments')) }}", + "values": [ + "credit_card", + "coupon", + "bank_transfer", + "gift_card" + ] + }, + "name": "accepted_values", + "namespace": null + }, + "unique_id": "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278", + "unrendered_config": { + "alias": "accepted_values_stg_payments_c7909fb19b1f0177c2bf99c7912f06ef" + } + }, + "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d": { + "alias": "not_null_customers_customer_id", + "attached_node": "model.jaffle_shop.customers", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "customer_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.8657112, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.customers" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.customers", + "fqn": [ + "jaffle_shop", + "not_null_customers_customer_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_customers_customer_id", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_customers_customer_id.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "customers", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "customer_id", + "model": "{{ get_where_subquery(ref('customers')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_orders_amount.106140f9fd": { + "alias": "not_null_orders_amount", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "amount", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.881857, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "not_null_orders_amount" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_orders_amount", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_orders_amount.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "amount", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_orders_amount.106140f9fd", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49": { + "alias": "not_null_orders_bank_transfer_amount", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "bank_transfer_amount", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.885247, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "not_null_orders_bank_transfer_amount" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_orders_bank_transfer_amount", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_orders_bank_transfer_amount.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "bank_transfer_amount", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625": { + "alias": "not_null_orders_coupon_amount", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "coupon_amount", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.884174, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "not_null_orders_coupon_amount" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_orders_coupon_amount", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_orders_coupon_amount.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "coupon_amount", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59": { + "alias": "not_null_orders_credit_card_amount", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "credit_card_amount", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.882951, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "not_null_orders_credit_card_amount" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_orders_credit_card_amount", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_orders_credit_card_amount.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "credit_card_amount", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_orders_customer_id.c5f02694af": { + "alias": "not_null_orders_customer_id", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "customer_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.869169, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "not_null_orders_customer_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_orders_customer_id", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_orders_customer_id.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "customer_id", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_orders_customer_id.c5f02694af", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a": { + "alias": "not_null_orders_gift_card_amount", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "gift_card_amount", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.886303, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "not_null_orders_gift_card_amount" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_orders_gift_card_amount", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_orders_gift_card_amount.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "gift_card_amount", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_orders_order_id.cf6c17daed": { + "alias": "not_null_orders_order_id", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "order_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.868075, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "not_null_orders_order_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_orders_order_id", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_orders_order_id.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "order_id", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_orders_order_id.cf6c17daed", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa": { + "alias": "not_null_stg_customers_customer_id", + "attached_node": "model.jaffle_shop.stg_customers", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "customer_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.891142, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.stg_customers" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_customers", + "fqn": [ + "jaffle_shop", + "staging", + "not_null_stg_customers_customer_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_stg_customers_customer_id", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_stg_customers_customer_id.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "stg_customers", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "customer_id", + "model": "{{ get_where_subquery(ref('stg_customers')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64": { + "alias": "not_null_stg_orders_order_id", + "attached_node": "model.jaffle_shop.stg_orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "order_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.893447, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.stg_orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_orders", + "fqn": [ + "jaffle_shop", + "staging", + "not_null_stg_orders_order_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_stg_orders_order_id", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_stg_orders_order_id.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "stg_orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "order_id", + "model": "{{ get_where_subquery(ref('stg_orders')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64", + "unrendered_config": {} + }, + "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075": { + "alias": "not_null_stg_payments_payment_id", + "attached_node": "model.jaffle_shop.stg_payments", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "payment_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.899931, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null" + ], + "nodes": [ + "model.jaffle_shop.stg_payments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_payments", + "fqn": [ + "jaffle_shop", + "staging", + "not_null_stg_payments_payment_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_stg_payments_payment_id", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "not_null_stg_payments_payment_id.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "stg_payments", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "payment_id", + "model": "{{ get_where_subquery(ref('stg_payments')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075", + "unrendered_config": {} + }, + "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2": { + "alias": "relationships_orders_customer_id__customer_id__ref_customers_", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "customer_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.870429, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_relationships", + "macro.dbt.get_where_subquery" + ], + "nodes": [ + "model.jaffle_shop.customers", + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "relationships_orders_customer_id__customer_id__ref_customers_" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "relationships_orders_customer_id__customer_id__ref_customers_", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "relationships_orders_customer_id__customer_id__ref_customers_.sql", + "raw_code": "{{ test_relationships(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "customers", + "package": null, + "version": null + }, + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "customer_id", + "field": "customer_id", + "model": "{{ get_where_subquery(ref('orders')) }}", + "to": "ref('customers')" + }, + "name": "relationships", + "namespace": null + }, + "unique_id": "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2", + "unrendered_config": {} + }, + "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1": { + "alias": "unique_customers_customer_id", + "attached_node": "model.jaffle_shop.customers", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "customer_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.864361, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_unique" + ], + "nodes": [ + "model.jaffle_shop.customers" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.customers", + "fqn": [ + "jaffle_shop", + "unique_customers_customer_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "unique_customers_customer_id", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "unique_customers_customer_id.sql", + "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "customers", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "customer_id", + "model": "{{ get_where_subquery(ref('customers')) }}" + }, + "name": "unique", + "namespace": null + }, + "unique_id": "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1", + "unrendered_config": {} + }, + "test.jaffle_shop.unique_orders_order_id.fed79b3a6e": { + "alias": "unique_orders_order_id", + "attached_node": "model.jaffle_shop.orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "order_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.866973, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_unique" + ], + "nodes": [ + "model.jaffle_shop.orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.orders", + "fqn": [ + "jaffle_shop", + "unique_orders_order_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "unique_orders_order_id", + "original_file_path": "models/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "unique_orders_order_id.sql", + "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "order_id", + "model": "{{ get_where_subquery(ref('orders')) }}" + }, + "name": "unique", + "namespace": null + }, + "unique_id": "test.jaffle_shop.unique_orders_order_id.fed79b3a6e", + "unrendered_config": {} + }, + "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada": { + "alias": "unique_stg_customers_customer_id", + "attached_node": "model.jaffle_shop.stg_customers", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "customer_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.88995, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_unique" + ], + "nodes": [ + "model.jaffle_shop.stg_customers" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_customers", + "fqn": [ + "jaffle_shop", + "staging", + "unique_stg_customers_customer_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "unique_stg_customers_customer_id", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "unique_stg_customers_customer_id.sql", + "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "stg_customers", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "customer_id", + "model": "{{ get_where_subquery(ref('stg_customers')) }}" + }, + "name": "unique", + "namespace": null + }, + "unique_id": "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada", + "unrendered_config": {} + }, + "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a": { + "alias": "unique_stg_orders_order_id", + "attached_node": "model.jaffle_shop.stg_orders", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "order_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.89237, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_unique" + ], + "nodes": [ + "model.jaffle_shop.stg_orders" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_orders", + "fqn": [ + "jaffle_shop", + "staging", + "unique_stg_orders_order_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "unique_stg_orders_order_id", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "unique_stg_orders_order_id.sql", + "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "stg_orders", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "order_id", + "model": "{{ get_where_subquery(ref('stg_orders')) }}" + }, + "name": "unique", + "namespace": null + }, + "unique_id": "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a", + "unrendered_config": {} + }, + "test.jaffle_shop.unique_stg_payments_payment_id.3744510712": { + "alias": "unique_stg_payments_payment_id", + "attached_node": "model.jaffle_shop.stg_payments", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "payment_id", + "columns": {}, + "compiled_path": null, + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "checksum": null, + "enforced": false + }, + "created_at": 1689766376.898259, + "database": null, + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_unique" + ], + "nodes": [ + "model.jaffle_shop.stg_payments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "file_key_name": "models.stg_payments", + "fqn": [ + "jaffle_shop", + "staging", + "unique_stg_payments_payment_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "unique_stg_payments_payment_id", + "original_file_path": "models/staging/schema.yml", + "package_name": "jaffle_shop", + "patch_path": null, + "path": "unique_stg_payments_payment_id.sql", + "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "stg_payments", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "cosmos_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "payment_id", + "model": "{{ get_where_subquery(ref('stg_payments')) }}" + }, + "name": "unique", + "namespace": null + }, + "unique_id": "test.jaffle_shop.unique_stg_payments_payment_id.3744510712", + "unrendered_config": {} + } + }, + "parent_map": { + "model.jaffle_shop.customers": [ + "model.jaffle_shop.stg_customers", + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments" + ], + "model.jaffle_shop.orders": [ + "model.jaffle_shop.stg_orders", + "model.jaffle_shop.stg_payments" + ], + "model.jaffle_shop.stg_customers": [ + "seed.jaffle_shop.raw_customers" + ], + "model.jaffle_shop.stg_orders": [ + "seed.jaffle_shop.raw_orders" + ], + "model.jaffle_shop.stg_payments": [ + "seed.jaffle_shop.raw_payments" + ], + "seed.jaffle_shop.raw_customers": [], + "seed.jaffle_shop.raw_orders": [], + "seed.jaffle_shop.raw_payments": [], + "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad": [ + "model.jaffle_shop.stg_orders" + ], + "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278": [ + "model.jaffle_shop.stg_payments" + ], + "test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d": [ + "model.jaffle_shop.customers" + ], + "test.jaffle_shop.not_null_orders_amount.106140f9fd": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.not_null_orders_customer_id.c5f02694af": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.not_null_orders_order_id.cf6c17daed": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa": [ + "model.jaffle_shop.stg_customers" + ], + "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64": [ + "model.jaffle_shop.stg_orders" + ], + "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075": [ + "model.jaffle_shop.stg_payments" + ], + "test.jaffle_shop.relationships_orders_customer_id__customer_id__ref_customers_.c6ec7f58f2": [ + "model.jaffle_shop.customers", + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1": [ + "model.jaffle_shop.customers" + ], + "test.jaffle_shop.unique_orders_order_id.fed79b3a6e": [ + "model.jaffle_shop.orders" + ], + "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada": [ + "model.jaffle_shop.stg_customers" + ], + "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a": [ + "model.jaffle_shop.stg_orders" + ], + "test.jaffle_shop.unique_stg_payments_payment_id.3744510712": [ + "model.jaffle_shop.stg_payments" + ] + }, + "selectors": {}, + "sources": {} +}