Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
* Fix Materialized View not dropped when a model's materialization is changed from materialized_view to view ([#516](https://github.com/ClickHouse/dbt-clickhouse/pull/516)).
* Ensure that temporary tables are not accessed with database clause ([#515](https://github.com/ClickHouse/dbt-clickhouse/pull/515)).
* Lock `dbt-adapters` version to avoid incompatibilities caused by unexpected version upgrades ([#530](https://github.com/ClickHouse/dbt-clickhouse/pull/530)).
* Alter destination table along with mv using `on_schema_changes` ([#534](https://github.com/ClickHouse/dbt-clickhouse/pull/534))

### Release [1.9.3], 2025-09-08

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,15 @@
select 1
{%- endcall %}

{%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}
{{ log('on_schema_change strategy for destination table of MV: ' + on_schema_change, info=True) }}
{%- if on_schema_change != 'ignore' -%}
{%- set column_changes = adapter.check_incremental_schema_changes(on_schema_change, existing_relation, sql) -%}
{% if column_changes %}
{% do clickhouse__apply_column_changes(column_changes, existing_relation) %}
{% set existing_relation = load_cached_relation(this) %}
{% endif %}
{%- endif %}
Comment on lines +89 to +97
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The code should check if existing_relation exists before attempting to apply schema changes. Without this check, the code might try to apply schema changes to a non-existent relation.

Suggested change
{%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}
{{ log('on_schema_change strategy for destination table of MV: ' + on_schema_change, info=True) }}
{%- if on_schema_change != 'ignore' -%}
{%- set column_changes = adapter.check_incremental_schema_changes(on_schema_change, existing_relation, sql) -%}
{% if column_changes %}
{% do clickhouse__apply_column_changes(column_changes, existing_relation) %}
{% set existing_relation = load_cached_relation(this) %}
{% endif %}
{%- endif %}
{%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}
{{ log('on_schema_change strategy for destination table of MV: ' + on_schema_change, info=True) }}
{%- if existing_relation is not none and on_schema_change != 'ignore' -%}
{%- set column_changes = adapter.check_incremental_schema_changes(on_schema_change, existing_relation, sql) -%}
{% if column_changes %}
{% do clickhouse__apply_column_changes(column_changes, existing_relation) %}
{% set existing_relation = load_cached_relation(this) %}
{% endif %}
{%- endif %}

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The default behavior in case of on_schema_change non existent is to ignore destination table change and just update the MV (current implementation)

-- try to alter view first to replace sql, else drop and create
{{ clickhouse__update_mvs(target_relation, cluster_clause, refreshable_clause, views) }}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
materialized='materialized_view' if var('run_type', '') != 'view_conversion' else 'view',
engine='MergeTree()',
order_by='(id)',
on_schema_change='sync_all_columns',
schema='catchup' if var('run_type', '') == 'catchup' else 'custom_schema',
**({'catchup': False} if var('run_type', '') == 'catchup' else {})
) }}
Expand All @@ -53,7 +54,8 @@
when name like 'Dade' and age != 11 then 'crash override'
when name like 'Kate' then 'acid burn'
else 'N/A'
end as hacker_alias
end as hacker_alias,
id as id2
from {{ source('raw', 'people') }}
where department = 'engineering'

Expand Down Expand Up @@ -213,6 +215,19 @@ def test_update_incremental(self, project):
)
assert len(result) == 2

# assert that the destination table is updated with the new column
table_description_after_update = project.run_sql(
f"DESCRIBE TABLE {schema}.hackers", fetch="all"
)
assert any(col[0] == "id2" and col[1] == "Int32" for col in table_description_after_update)

# run again without extended schema, to make sure table is updated back without the id2 column
run_dbt()
table_description_after_revert_update = project.run_sql(
f"DESCRIBE TABLE {schema}.hackers", fetch="all"
)
assert not any(col[0] == "id2" for col in table_description_after_revert_update)

def test_update_full_refresh(self, project):
schema = quote_identifier(project.test_schema + "_custom_schema")
# create our initial materialized view
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
materialized='materialized_view',
engine='MergeTree()',
order_by='(id)',
on_schema_change='sync_all_columns',
schema='custom_schema_for_multiple_mv',
) }}

Expand Down Expand Up @@ -70,7 +71,8 @@
when name like 'Dade' and age != 11 then 'crash override'
when name like 'Kate' then 'acid burn'
else 'N/A'
end as hacker_alias
end as hacker_alias,
id as id2
from {{ source('raw', 'people') }}
where department = 'engineering'
--mv1:end
Expand All @@ -82,7 +84,8 @@
id,
name,
-- sales people are not cool enough to have a hacker alias
'N/A' as hacker_alias
'N/A' as hacker_alias,
id as id2
from {{ source('raw', 'people') }}
where department = 'sales'
--mv2:end
Expand Down Expand Up @@ -219,6 +222,17 @@ def test_update_incremental(self, project):
assert result[0][0] == "crash_override"
assert result[1][0] == "zero cool"

# assert that the destination table is updated with the new column
table_description_after_update = project.run_sql(f"DESCRIBE {schema}.hackers", fetch="all")
assert any(col[0] == "id2" and col[1] == "Int32" for col in table_description_after_update)

# run again without extended schema, to make sure table is updated back without the id2 column
run_dbt()
table_description_after_revert_update = project.run_sql(
f"DESCRIBE TABLE {schema}.hackers", fetch="all"
)
assert not any(col[0] == "id2" for col in table_description_after_revert_update)

def test_update_full_refresh(self, project):
schema = quote_identifier(project.test_schema + "_custom_schema_for_multiple_mv")
# create our initial materialized view
Expand Down