Skip to content

80 additional model exists verification #81

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 30, 2024
Merged

80 additional model exists verification #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
04251d5
Added test case for missing parent table and selecting child
sfc-gh-dflippo Sep 30, 2024
01a84e2
Added logic to skip constraints when tables don't exist
sfc-gh-dflippo Sep 30, 2024
e03845a
Updated to the latest version of dbt Utils for integration tests
sfc-gh-dflippo Sep 30, 2024
3fb434e
Correctly skipping constraints when there was a failure.
sfc-gh-dflippo Sep 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dbt_project.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@

name: 'dbt_constraints'
version: '1.0.3'
version: '1.0.4'
config-version: 2

# These macros depend on the results and graph objects in dbt >=0.19.0
Expand Down
2 changes: 2 additions & 0 deletions integration_tests/dbt_project.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ vars:
dbt_constraints_sources_uk_enabled: true
dbt_constraints_sources_fk_enabled: true

on-run-start:
- "drop table if exists dim_orders"

models:
+materialized: table
Expand Down
2 changes: 1 addition & 1 deletion integration_tests/packages.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@ packages:

# We will test unique keys based on dbt_utils.unique_combination_of_columns
- package: dbt-labs/dbt_utils
version: [">=1.2.0"]
version: [">=1.3.0"]

- local: ../
130 changes: 79 additions & 51 deletions macros/create_constraints.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,16 +174,15 @@
{#- This macro checks if a test or its model is selected -#}
{%- macro test_selected(test_model) -%}

{%- if test_model.unique_id and test_model.unique_id in selected_resources -%}
{%- if test_model.unique_id in selected_resources -%}
{{ return("TEST_SELECTED") }}
{%- endif -%}
{%- if test_model.attached_node and test_model.attached_node in selected_resources -%} -%}
{%- if test_model.attached_node in selected_resources -%} -%}
{{ return("MODEL_SELECTED") }}
{%- endif -%}

{#- Check if a PK/UK should be created because it is referenced by a selected FK -#}
{%- if test_model.test_metadata
and test_model.test_metadata.name in ("primary_key", "unique_key", "unique_combination_of_columns", "unique") -%}
{%- if test_model.test_metadata.name in ("primary_key", "unique_key", "unique_combination_of_columns", "unique") -%}
{%- set pk_test_args = test_model.test_metadata.kwargs -%}
{%- set pk_test_columns = [] -%}
{%- if pk_test_args.column_names -%}
Expand Down Expand Up @@ -230,7 +229,10 @@
{%- for res in results
if res.node.config.materialized == "test"
and res.node.unique_id == test_model.unique_id -%}
{%- if res.failures > 0 -%}
{%- if res.failures == None -%}
{#- Set '' if we do not know if there is a test failure -#}
{{ return('') }}
{%- elif res.failures > 0 -%}
{#- Set NORELY if there is a test failure -#}
{{ return('NORELY') }}
{%- elif res.failures == 0 -%}
Expand Down Expand Up @@ -261,10 +263,17 @@
{#- This macro is called internally and passed which constraint types to create. -#}
{%- macro create_constraints_by_type(constraint_types, quote_columns, lookup_cache) -%}

{#- Loop through the metadata and find all tests that match the constraint_types -#}
{#- Loop through the metadata and find all tests that match the constraint_types and have all the fields we check for tests -#}
{%- for test_model in graph.nodes.values() | selectattr("resource_type", "equalto", "test")
if test_model.test_metadata
and test_model.test_metadata.kwargs
and test_model.test_metadata.name
and test_model.test_metadata.name is in( constraint_types )
and test_model.unique_id
and test_model.attached_node
and test_model.depends_on
and test_model.depends_on.nodes
and test_model.config
and test_model.config.enabled
and test_model.config.get("dbt_constraints_enabled", true) -%}

Expand Down Expand Up @@ -300,7 +309,8 @@
{#- Find the table models that are referenced by this test. -#}
{%- for table_node in test_model.depends_on.nodes -%}
{%- for node in graph.nodes.values() | selectattr("unique_id", "equalto", table_node)
if node.config.get("materialized", "other") not in ("view", "ephemeral", "dynamic_table")
if node.config
and node.config.get("materialized", "other") not in ("view", "ephemeral", "dynamic_table")
and ( node.resource_type in ("model", "snapshot", "seed")
or ( node.resource_type == "source" and var('dbt_constraints_sources_enabled', false)
and ( ( var('dbt_constraints_sources_pk_enabled', false) and test_name in("primary_key") )
Expand Down Expand Up @@ -340,21 +350,25 @@
) }}
{%- endif -%}

{%- set table_relation = api.Relation.create(
{%- set table_relation = adapter.get_relation(
database=table_models[0].database,
schema=table_models[0].schema,
identifier=table_models[0].alias ) -%}
{%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%}
{%- if test_name == "primary_key" -%}
{%- if dbt_constraints.adapter_supports_rely_norely("not_null") == true -%}
{%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%}
{%- if table_relation and table_relation.is_table -%}
{%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%}
{%- if test_name == "primary_key" -%}
{%- if dbt_constraints.adapter_supports_rely_norely("not_null") == true -%}
{%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%}
{%- endif -%}
{%- do dbt_constraints.create_primary_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%}
{%- else -%}
{%- do dbt_constraints.create_unique_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%}
{%- endif -%}
{%- do dbt_constraints.create_primary_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%}
{%- else -%}
{%- do dbt_constraints.create_unique_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%}
{%- do log("Skipping primary/unique key because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%}
{%- endif -%}
{%- else -%}
{%- do log("Skipping primary/unique key because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%}
{%- do log("Skipping primary/unique key because the table was not found in the database: " ~ table_models[0].name, info=true) -%}
{%- endif -%}

{#- We only create FK if there are two models referenced by the test
Expand All @@ -367,50 +381,60 @@

{%- if fk_model and pk_model -%}

{%- set fk_table_relation = api.Relation.create(
{%- set fk_table_relation = adapter.get_relation(
database=fk_model.database,
schema=fk_model.schema,
identifier=fk_model.alias) -%}

{%- set pk_table_relation = api.Relation.create(
{%- set pk_table_relation = adapter.get_relation(
database=pk_model.database,
schema=pk_model.schema,
identifier=pk_model.alias) -%}

{# Attempt to identify parameters we can use for the column names #}
{%- set pk_column_names = [] -%}
{%- if test_parameters.pk_column_names -%}
{%- set pk_column_names = test_parameters.pk_column_names -%}
{%- elif test_parameters.field -%}
{%- set pk_column_names = [test_parameters.field] -%}
{%- elif test_parameters.pk_column_name -%}
{%- set pk_column_names = [test_parameters.pk_column_name] -%}
{%- else -%}
{{ exceptions.raise_compiler_error(
"`pk_column_names`, `pk_column_name`, or `field` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters
) }}
{%- endif -%}
{%- if fk_table_relation and pk_table_relation and fk_table_relation.is_table and pk_table_relation.is_table-%}
{# Attempt to identify parameters we can use for the column names #}
{%- set pk_column_names = [] -%}
{%- if test_parameters.pk_column_names -%}
{%- set pk_column_names = test_parameters.pk_column_names -%}
{%- elif test_parameters.field -%}
{%- set pk_column_names = [test_parameters.field] -%}
{%- elif test_parameters.pk_column_name -%}
{%- set pk_column_names = [test_parameters.pk_column_name] -%}
{%- else -%}
{{ exceptions.raise_compiler_error(
"`pk_column_names`, `pk_column_name`, or `field` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters
) }}
{%- endif -%}

{%- set fk_column_names = [] -%}
{%- if test_parameters.fk_column_names -%}
{%- set fk_column_names = test_parameters.fk_column_names -%}
{%- elif test_parameters.column_name -%}
{%- set fk_column_names = [test_parameters.column_name] -%}
{%- elif test_parameters.fk_column_name -%}
{%- set fk_column_names = [test_parameters.fk_column_name] -%}
{%- else -%}
{{ exceptions.raise_compiler_error(
"`fk_column_names`, `fk_column_name`, or `column_name` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters
) }}
{%- endif -%}
{%- set fk_column_names = [] -%}
{%- if test_parameters.fk_column_names -%}
{%- set fk_column_names = test_parameters.fk_column_names -%}
{%- elif test_parameters.column_name -%}
{%- set fk_column_names = [test_parameters.column_name] -%}
{%- elif test_parameters.fk_column_name -%}
{%- set fk_column_names = [test_parameters.fk_column_name] -%}
{%- else -%}
{{ exceptions.raise_compiler_error(
"`fk_column_names`, `fk_column_name`, or `column_name` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters
) }}
{%- endif -%}

{%- if not dbt_constraints.table_columns_all_exist(pk_table_relation, pk_column_names, lookup_cache) -%}
{%- do log("Skipping foreign key because a physical column was not found on the pk table: " ~ pk_model.name ~ " " ~ pk_column_names, info=true) -%}
{%- elif not dbt_constraints.table_columns_all_exist(fk_table_relation, fk_column_names, lookup_cache) -%}
{%- do log("Skipping foreign key because a physical column was not found on the fk table: " ~ fk_model.name ~ " " ~ fk_column_names, info=true) -%}
{%- if not dbt_constraints.table_columns_all_exist(pk_table_relation, pk_column_names, lookup_cache) -%}
{%- do log("Skipping foreign key because a physical column was not found on the pk table: " ~ pk_model.name ~ " " ~ pk_column_names, info=true) -%}
{%- elif not dbt_constraints.table_columns_all_exist(fk_table_relation, fk_column_names, lookup_cache) -%}
{%- do log("Skipping foreign key because a physical column was not found on the fk table: " ~ fk_model.name ~ " " ~ fk_column_names, info=true) -%}
{%- else -%}
{%- do dbt_constraints.create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%}
{%- endif -%}
{%- else -%}
{%- do dbt_constraints.create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%}
{%- if fk_model == None or not fk_table_relation.is_table -%}
{%- do log("Skipping foreign key to " ~ pk_model.alias ~ " because the child table was not found in the database: " ~ fk_model.alias, info=true) -%}
{%- endif -%}
{%- if pk_model == None or not pk_model.is_table -%}
{%- do log("Skipping foreign key on " ~ fk_model.alias ~ " because the parent table was not found in the database: " ~ pk_model.alias, info=true) -%}
{%- endif -%}
{%- endif -%}

{%- else -%}
{%- do log("Skipping foreign key because a we couldn't find the child table: model=" ~ test_model.attached_node ~ " or source", info=true) -%}
{%- endif -%}
Expand All @@ -434,15 +458,19 @@
) }}
{%- endif -%}

{%- set table_relation = api.Relation.create(
{%- set table_relation = adapter.get_relation(
database=table_models[0].database,
schema=table_models[0].schema,
identifier=table_models[0].alias ) -%}

{%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%}
{%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%}
{%- if table_relation and table_relation.is_table -%}
{%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%}
{%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%}
{%- else -%}
{%- do log("Skipping not null constraint because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%}
{%- endif -%}
{%- else -%}
{%- do log("Skipping not null constraint because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%}
{%- do log("Skipping not null constraint because the table was not found in the database: " ~ table_models[0].name, info=true) -%}
{%- endif -%}

{%- endif -%}
Expand Down