Skip to content

Commit e7f0e60

Browse files
0xRobin0xRobin
authored
Seed and generic test support for DuneSQL (#3676)
* seed test support for dunesql * forgot one _legacy * add comment
1 parent 4161a29 commit e7f0e60

File tree

195 files changed

+652
-430
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

195 files changed

+652
-430
lines changed

‎macros/dune/create_bindings.sql‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{#
2+
This macro is used in https://github.com/starburstdata/dbt-trino/blob/master/dbt/include/trino/macros/materializations/seeds/helpers.sql
3+
We need to override the type bindings to support varbinary hex addresses and hashes in seeds.
4+
#}
5+
6+
7+
{% macro create_bindings(row, types) %}
8+
{% set values = [] %}
9+
{% set re = modules.re %}
10+
11+
{%- for item in row -%}
12+
{%- set type = types[loop.index0] -%}
13+
{%- set match_type = re.match("(\w+)(\(.*\))?", type) -%}
14+
{%- if item is not none and 'varbinary' in type.lower() -%}
15+
{%- do values.append((none, item )) -%}
16+
{%- elif item is not none and item is string and 'interval' in match_type.group(1) -%}
17+
{%- do values.append((none, match_type.group(1).upper() ~ " " ~ item)) -%}
18+
{%- elif item is not none and item is string and 'varchar' not in type.lower() -%}
19+
{%- do values.append((none, match_type.group(1).upper() ~ " '" ~ item ~ "'")) -%}
20+
{%- elif item is not none and 'varchar' in type.lower() -%}
21+
{%- do values.append((get_binding_char(), item|string())) -%}
22+
{%- else -%}
23+
{%- do values.append((get_binding_char(), item)) -%}
24+
{% endif -%}
25+
{%- endfor -%}
26+
{{ return(values) }}
27+
{% endmacro %}

‎macros/test-helpers/check_seed_macro.sql‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,10 @@
5454
matching_count_test as (
5555
select
5656
'matched records count' as test_description,
57-
count(model_{{seed_matching_columns[0]}}) as `result_model`,
58-
1 as `expected_seed`,
57+
-- these are cast to varchar to unify column types, note this is only for displaying them in the test results
58+
cast(count(model_{{seed_matching_columns[0]}}) as varchar) as result_model,
59+
cast(1 as varchar) as expected_seed,
60+
(count(model_{{seed_matching_columns[0]}}) = 1) as equality_check,
5961
{%- for column_name in seed_matching_columns %}
6062
seed_{{column_name}} as {{column_name}}{% if not loop.last %},{% endif %}
6163
{% endfor -%}
@@ -71,16 +73,14 @@
7173
{%- for checked_column in seed_check_columns %}
7274
select
7375
'equality test: {{checked_column}}' as test_description
74-
,test.*
75-
from (
76-
select
77-
model_{{checked_column}} as `result_model`,
78-
seed_{{checked_column}} as `expected_seed`,
79-
{%- for column_name in seed_matching_columns %}
80-
seed_{{column_name}} {% if not loop.last %},{% endif %}
81-
{% endfor -%}
82-
from matched_records
83-
) test
76+
-- these are cast to varchar to unify column types, note this is only for displaying them in the test results
77+
cast(model_{{checked_column}} as varchar) as result_model,
78+
cast(seed_{{checked_column}} as varchar) as expected_seed,
79+
(model_{{checked_column}} = seed_{{checked_column}}) as equality_check,
80+
{%- for column_name in seed_matching_columns %}
81+
seed_{{column_name}} as {{column_name}}{% if not loop.last %},{% endif %}
82+
{% endfor -%}
83+
from matched_records
8484
{%- if not loop.last %}
8585
UNION ALL
8686
{% endif -%}
@@ -95,5 +95,5 @@
9595
select *
9696
from equality_tests
9797
) all
98-
where `result_model` != `expected_seed`
98+
where equality_check != true
9999
{% endmacro %}

‎macros/test-helpers/check_seed_macro_legacy.sql‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
-- this macro is used in generic tests that check a model for every row in a seed file.
2+
-- you need to specify the matching columns and the columns to check for equality.
3+
-- filter: dictionary filter of column:value that is applied to the seed file
4+
5+
{% macro check_seed_macro_legacy(model, seed_file, seed_matching_columns=[], seed_check_columns=[], filter=None) %}
6+
7+
with matched_records as (
8+
select
9+
{%- for column_name in seed_matching_columns %}
10+
seed.{{column_name}} as seed_{{column_name}},
11+
model_sample.{{column_name}} as model_{{column_name}},
12+
{% endfor -%}
13+
{%- for column_name in seed_check_columns %}
14+
seed.{{column_name}} as seed_{{column_name}},
15+
model_sample.{{column_name}} as model_{{column_name}} {% if not loop.last %},{% endif %}
16+
{% endfor -%}
17+
from {{seed_file}} seed
18+
left join (
19+
select
20+
{%- for column_name in seed_matching_columns %}
21+
model.{{column_name}},
22+
{% endfor -%}
23+
{%- for column_name in seed_check_columns %}
24+
model.{{column_name}} {% if not loop.last %},{% endif %}
25+
{% endfor -%}
26+
from {{seed_file}} seed
27+
inner join {{model}} model
28+
ON 1=1
29+
{%- for column_name in seed_matching_columns %}
30+
{% if column_name == 'trace_address' %}
31+
AND COALESCE(CAST(split(seed.{{column_name}}, ',') as array<bigint>), ARRAY()) = model.{{column_name}}
32+
{% else %}
33+
AND seed.{{column_name}} = model.{{column_name}}
34+
{% endif %}
35+
{% endfor -%}
36+
) model_sample
37+
ON 1=1
38+
{%- for column_name in seed_matching_columns %}
39+
{% if column_name == 'trace_address' %}
40+
AND COALESCE(CAST(split(seed.{{column_name}}, ',') as array<bigint>), ARRAY()) = model_sample.{{column_name}}
41+
{% else %}
42+
AND seed.{{column_name}} = model_sample.{{column_name}}
43+
{% endif %}
44+
{% endfor -%}
45+
WHERE 1=1
46+
{%- if filter is not none %}
47+
{%- for col, val in filter.items() %}
48+
{% if val is not none %} AND seed.{{col}} = '{{val}}' {% endif %}
49+
{% endfor -%}
50+
{% endif -%}
51+
),
52+
53+
-- check if the matching columns return singular results
54+
matching_count_test as (
55+
select
56+
'matched records count' as test_description,
57+
count(model_{{seed_matching_columns[0]}}) as `result_model`,
58+
1 as `expected_seed`,
59+
{%- for column_name in seed_matching_columns %}
60+
seed_{{column_name}} as {{column_name}}{% if not loop.last %},{% endif %}
61+
{% endfor -%}
62+
from matched_records
63+
GROUP BY
64+
{%- for column_name in seed_matching_columns %}
65+
seed_{{column_name}} {% if not loop.last %},{% endif %}
66+
{% endfor -%}
67+
) ,
68+
69+
equality_tests as
70+
(
71+
{%- for checked_column in seed_check_columns %}
72+
select
73+
'equality test: {{checked_column}}' as test_description
74+
,test.*
75+
from (
76+
select
77+
model_{{checked_column}} as `result_model`,
78+
seed_{{checked_column}} as `expected_seed`,
79+
{%- for column_name in seed_matching_columns %}
80+
seed_{{column_name}} {% if not loop.last %},{% endif %}
81+
{% endfor -%}
82+
from matched_records
83+
) test
84+
{%- if not loop.last %}
85+
UNION ALL
86+
{% endif -%}
87+
{% endfor -%}
88+
)
89+
90+
91+
select * from (
92+
select *
93+
from matching_count_test
94+
union all
95+
select *
96+
from equality_tests
97+
) all
98+
where `result_model` != `expected_seed`
99+
{% endmacro %}

‎macros/test-helpers/test_helpers_schema.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
version: 2
22

33
macros:
4+
- name: check_seed_macro_legacy
5+
description: "Macro to be used in generic seed check tests. It checks if every row in a seed file is present in the model"
46
- name: check_seed_macro
57
description: "Macro to be used in generic seed check tests. It checks if every row in a seed file is present in the model"

‎models/_sector/nft/trades/ethereum/platforms/_schema.yml‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ models:
1616
- equal_rowcount_with_sources:
1717
evt_sources:
1818
- source('archipelago_ethereum','ArchipelagoMarket_evt_Trade')
19-
- check_seed:
19+
- check_seed_legacy:
2020
seed_file: ref('archipelago_ethereum_base_trades_seed')
2121
match_columns:
2222
- block_number
@@ -44,7 +44,7 @@ models:
4444
tests:
4545
- dbt_utils.unique_combination_of_columns:
4646
combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ]
47-
- check_seed:
47+
- check_seed_legacy:
4848
seed_file: ref('superrare_ethereum_base_trades_seed')
4949
match_columns:
5050
- block_number
@@ -78,7 +78,7 @@ models:
7878
- source('foundation_ethereum','market_evt_BuyPriceAccepted')
7979
- source('foundation_ethereum','market_evt_OfferAccepted')
8080
- source('foundation_ethereum','market_evt_PrivateSaleFinalized')
81-
- check_seed:
81+
- check_seed_legacy:
8282
seed_file: ref('foundation_ethereum_base_trades_seed')
8383
match_columns:
8484
- block_number
@@ -109,7 +109,7 @@ models:
109109
error_if: ">1"
110110
evt_sources:
111111
- source('cryptopunks_ethereum','CryptoPunksMarket_evt_PunkBought')
112-
- check_seed:
112+
- check_seed_legacy:
113113
seed_file: ref('cryptopunks_ethereum_base_trades_seed')
114114
match_columns:
115115
- block_number
@@ -136,7 +136,7 @@ models:
136136
tests:
137137
- dbt_utils.unique_combination_of_columns:
138138
combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ]
139-
- check_seed:
139+
- check_seed_legacy:
140140
seed_file: ref('blur_ethereum_base_trades_seed')
141141
match_columns:
142142
- block_number
@@ -169,7 +169,7 @@ models:
169169
- source('element_ex_ethereum','OrdersFeature_evt_ERC721BuyOrderFilled')
170170
- source('element_ex_ethereum','OrdersFeature_evt_ERC1155SellOrderFilled')
171171
- source('element_ex_ethereum','OrdersFeature_evt_ERC1155BuyOrderFilled')
172-
- check_seed:
172+
- check_seed_legacy:
173173
seed_file: ref('element_ethereum_base_trades_seed')
174174
match_columns:
175175
- block_number
@@ -199,7 +199,7 @@ models:
199199
- equal_rowcount_with_sources:
200200
evt_sources:
201201
- source('x2y2_ethereum','X2Y2_r1_evt_EvInventory')
202-
- check_seed:
202+
- check_seed_legacy:
203203
seed_file: ref('x2y2_ethereum_base_trades_seed')
204204
match_columns:
205205
- block_number
@@ -230,7 +230,7 @@ models:
230230
evt_sources:
231231
- source('looksrare_ethereum','LooksRareExchange_evt_TakerAsk')
232232
- source('looksrare_ethereum','LooksRareExchange_evt_TakerBid')
233-
- check_seed:
233+
- check_seed_legacy:
234234
seed_file: ref('looksrare_v1_ethereum_base_trades_seed')
235235
match_columns:
236236
- block_number
@@ -261,7 +261,7 @@ models:
261261
evt_sources:
262262
- source('looksrare_v2_ethereum','LooksRareProtocol_evt_TakerAsk')
263263
- source('looksrare_v2_ethereum','LooksRareProtocol_evt_TakerBid')
264-
- check_seed:
264+
- check_seed_legacy:
265265
seed_file: ref('looksrare_v2_ethereum_base_trades_seed')
266266
match_columns:
267267
- block_number
@@ -287,7 +287,7 @@ models:
287287
tests:
288288
- dbt_utils.unique_combination_of_columns:
289289
combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ]
290-
- check_seed:
290+
- check_seed_legacy:
291291
seed_file: ref('zora_v1_ethereum_base_trades_seed')
292292
match_columns:
293293
- block_number
@@ -317,7 +317,7 @@ models:
317317
- equal_rowcount_with_sources:
318318
evt_sources:
319319
- source('zora_ethereum','AuctionHouse_evt_AuctionEnded')
320-
- check_seed:
320+
- check_seed_legacy:
321321
seed_file: ref('zora_v2_ethereum_base_trades_seed')
322322
match_columns:
323323
- block_number
@@ -357,7 +357,7 @@ models:
357357
- source('zora_v3_ethereum','ReserveAuctionListingErc20_evt_AuctionEnded')
358358
- source('zora_v3_ethereum','AsksPrivateEth_evt_AskFilled')
359359
- source('zora_v3_ethereum','AsksCoreEth_evt_AskFilled')
360-
- check_seed:
360+
- check_seed_legacy:
361361
seed_file: ref('zora_v3_ethereum_base_trades_seed')
362362
match_columns:
363363
- block_number
@@ -384,7 +384,7 @@ models:
384384
tests:
385385
- dbt_utils.unique_combination_of_columns:
386386
combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ]
387-
- check_seed:
387+
- check_seed_legacy:
388388
seed_file: ref('sudoswap_ethereum_base_trades_seed')
389389
match_columns:
390390
- block_number
@@ -411,7 +411,7 @@ models:
411411
tests:
412412
- dbt_utils.unique_combination_of_columns:
413413
combination_of_columns: [ 'block_number','tx_hash','sub_tx_trade_id' ]
414-
- check_seed:
414+
- check_seed_legacy:
415415
seed_file: ref('collectionswap_ethereum_base_trades_seed')
416416
match_columns:
417417
- block_number

‎models/_sector/nft/trades/old/platforms/_schema.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ models:
183183
- token_id
184184
- sub_type
185185
- sub_idx
186-
- check_seed:
186+
- check_seed_legacy:
187187
seed_file: ref('nftearth_events_seed')
188188
filter:
189189
blockchain: optimism
@@ -223,7 +223,7 @@ models:
223223
combination_of_columns:
224224
- block_date
225225
- unique_trade_id
226-
- check_seed:
226+
- check_seed_legacy:
227227
seed_file: ref('pancakeswap_nft_trades_samples')
228228
match_columns:
229229
- block_time

‎models/_sector/nft/trades/old/platforms/decentraland_polygon_schema.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ models:
1515
combination_of_columns:
1616
- block_time
1717
- unique_trade_id
18-
- check_seed:
18+
- check_seed_legacy:
1919
seed_file: ref('decentraland_polygon_sample_trades')
2020
match_columns:
2121
- block_number
@@ -142,4 +142,4 @@ models:
142142
name: unique_trade_id
143143
description: "Unique trade ID"
144144
tests:
145-
- unique
145+
- unique

‎models/_sector/nft/trades/old/platforms/fractal_polygon_schema.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ models:
1515
combination_of_columns:
1616
- block_time
1717
- unique_trade_id
18-
- check_seed:
18+
- check_seed_legacy:
1919
seed_file: ref('fractal_polygon_sample_trades')
2020
match_columns:
2121
- block_number

‎models/_sector/nft/trades/old/platforms/liquidifty_bnb_schema.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ models:
1111
tags: ['liquidifty', 'nft', 'trades']
1212
description: "NFT trades on liquidifty on BNB blockchain"
1313
tests:
14-
- check_seed:
14+
- check_seed_legacy:
1515
seed_file: ref('liquidifty_bnb_nft_trades_samples')
1616
match_columns:
1717
- block_number

‎models/_sector/nft/trades/old/platforms/liquidifty_ethereum_schema.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ models:
1111
tags: ['liquidifty', 'nft', 'trades']
1212
description: "NFT trades on liquidifty on ethereum blockchain"
1313
tests:
14-
- check_seed:
14+
- check_seed_legacy:
1515
seed_file: ref('liquidifty_ethereum_nft_trades_samples')
1616
match_columns:
1717
- block_number

0 commit comments

Comments
 (0)