migrate foundation_ethereum_base_trades

.github/workflows/dbt_slim_ci.yml

@@ -67,7 +67,7 @@ jobs:
         run: "dbt compile $PROFILE $COMPILE_TAG"
 
       - name: dbt seed
-        run: "dbt seed $PROFILE --select state:modified$TAG --state ."
+        run: "dbt seed $PROFILE --select state:modified$TAG --exclude $EXCLUDE tag:prod_exclude --state ."
 
       - name: dbt run initial model(s)
         run: "dbt -x run $PROFILE --select state:modified$TAG --exclude $EXCLUDE tag:prod_exclude --defer --state ."

macros/dune/source.sql

@@ -1,9 +1,9 @@
 {% macro source(source_name, table_name) %}
   {% set rel = builtins.source(source_name, table_name) %}
-  {%- if 'dunesql' in model.config.get("tags") -%}
+  {%- if target.type == 'trino'-%}
     {% set newrel = rel.replace_path(database="delta_prod") %}
     {% do return(newrel) %}
   {%- else -%}
     {% do return(rel) %}
   {%- endif -%}
-{% endmacro %}
+{% endmacro %}

macros/test-helpers/check_seed_macro.sql

@@ -72,7 +72,7 @@
     (
         {%- for checked_column in seed_check_columns %}
         select
-            'equality test: {{checked_column}}' as test_description
+            'equality test: {{checked_column}}' as test_description,
             -- these are cast to varchar to unify column types, note this is only for displaying them in the test results
             cast(model_{{checked_column}} as varchar) as result_model,
             cast(seed_{{checked_column}} as varchar) as expected_seed,

models/_sector/nft/trades/ethereum/platforms/_schema.yml

@@ -78,7 +78,7 @@ models:
            - source('foundation_ethereum','market_evt_BuyPriceAccepted')
            - source('foundation_ethereum','market_evt_OfferAccepted')
            - source('foundation_ethereum','market_evt_PrivateSaleFinalized')
-     - check_seed_legacy:
+     - check_seed:
          seed_file: ref('foundation_ethereum_base_trades_seed')
          match_columns:
            - block_number

models/_sector/nft/trades/ethereum/platforms/foundation_ethereum_base_trades.sql

@@ -1,6 +1,7 @@
 {{ config(
+    tags = ['dunesql'],
     schema = 'foundation_ethereum',
-    alias ='base_trades',
+    alias = alias('base_trades'),
     partition_by = ['block_date'],
     materialized = 'incremental',
     file_format = 'delta',
@@ -16,88 +17,88 @@ WITH all_foundation_trades AS (
     , f.evt_block_number AS block_number
     , c.tokenId AS nft_token_id
     , 'Auction Settled' AS trade_category
-    , case when (f.sellerRev = 0 and cast(f.creatorRev as decimal(38)) > 0 ) then 'primary' else 'secondary' end as trade_type
+    , case when (f.sellerRev = UINT256 '0' and f.creatorRev > UINT256 '0' ) then 'primary' else 'secondary' end as trade_type
     , f.seller
     , f.bidder AS buyer
     , f.creatorRev+f.totalFees+f.sellerRev AS price_raw
     , f.contract_address AS project_contract_address
     , c.nftContract AS nft_contract_address
     , f.evt_tx_hash AS tx_hash
     , f.totalFees AS platform_fee_amount_raw
-    , case when (f.sellerRev = 0 and cast(f.creatorRev as decimal(38)) > 0 ) then 0 else f.creatorRev end AS royalty_fee_amount_raw
+    , case when (f.sellerRev = UINT256 '0' and f.creatorRev > UINT256 '0') then UINT256 '0' else f.creatorRev end AS royalty_fee_amount_raw
     , f.evt_index as sub_tx_trade_id
     FROM {{ source('foundation_ethereum','market_evt_ReserveAuctionFinalized') }} f
     INNER JOIN {{ source('foundation_ethereum','market_evt_ReserveAuctionCreated') }} c ON c.auctionId=f.auctionId AND c.evt_block_time<=f.evt_block_time
      {% if is_incremental() %} -- this filter will only be applied on an incremental run
-     WHERE f.evt_block_time >= date_trunc("day", now() - interval '1 week')
+     WHERE f.evt_block_time >= date_trunc('day', now() - interval '7' day)
      {% else %}
-     WHERE f.evt_block_time >= '{{project_start_date}}'
+     WHERE f.evt_block_time >= TIMESTAMP '{{project_start_date}}'
      {% endif %}
     UNION ALL
     SELECT
      evt_block_time AS block_time
     , evt_block_number AS block_number
     , tokenId AS nft_token_id
     , 'Buy' AS trade_category
-    , case when (sellerRev = 0 and cast(creatorRev as decimal(38)) > 0 ) then 'primary' else 'secondary' end as trade_type
+    , case when (sellerRev = UINT256 '0' and creatorRev > UINT256 '0' ) then 'primary' else 'secondary' end as trade_type
     , seller
     , buyer
     , creatorRev+totalFees+sellerRev AS price_raw
     , contract_address AS project_contract_address
     , nftContract AS nft_contract_address
     , evt_tx_hash AS tx_hash
     , totalFees AS platform_fee_amount_raw
-    , case when (sellerRev = 0 and cast(creatorRev as decimal(38)) > 0 ) then 0 else creatorRev end AS royalty_fee_amount_raw
+    , case when (sellerRev = UINT256 '0' and creatorRev > UINT256 '0' ) then UINT256 '0' else creatorRev end AS royalty_fee_amount_raw
     , evt_index as sub_tx_trade_id
     FROM {{ source('foundation_ethereum','market_evt_BuyPriceAccepted') }} f
      {% if is_incremental() %} -- this filter will only be applied on an incremental run
-     WHERE f.evt_block_time >= date_trunc("day", now() - interval '1 week')
+     WHERE f.evt_block_time >= date_trunc('day', now() - interval '7' day)
      {% else %}
-     WHERE f.evt_block_time >= '{{project_start_date}}'
+     WHERE f.evt_block_time >= TIMESTAMP '{{project_start_date}}'
      {% endif %}
     UNION ALL
     SELECT
      evt_block_time AS block_time
     , evt_block_number AS block_number
     , tokenId AS nft_token_id
     , 'Sell' AS trade_category
-    , case when (sellerRev = 0 and cast(creatorRev as decimal(38)) > 0 ) then 'primary' else 'secondary' end as trade_type
+    , case when (sellerRev = UINT256 '0' and creatorRev > UINT256 '0') then 'primary' else 'secondary' end as trade_type
     , seller
     , buyer
     , creatorRev+totalFees+sellerRev AS price_raw
     , contract_address AS project_contract_address
     , nftContract AS nft_contract_address
     , evt_tx_hash AS tx_hash
     , totalFees AS platform_fee_amount_raw
-    , case when (sellerRev = 0 and cast(creatorRev as decimal(38)) > 0 ) then 0 else creatorRev end AS royalty_fee_amount_raw
+    , case when (sellerRev = UINT256 '0' and creatorRev > UINT256 '0' ) then UINT256 '0' else creatorRev end AS royalty_fee_amount_raw
     , evt_index as sub_tx_trade_id
     FROM {{ source('foundation_ethereum','market_evt_OfferAccepted') }} f
      {% if is_incremental() %} -- this filter will only be applied on an incremental run
-     WHERE f.evt_block_time >= date_trunc("day", now() - interval '1 week')
+     WHERE f.evt_block_time >= date_trunc('day', now() - interval '7' day)
      {% else %}
-     WHERE f.evt_block_time >= '{{project_start_date}}'
+     WHERE f.evt_block_time >= TIMESTAMP '{{project_start_date}}'
      {% endif %}
     UNION ALL
     SELECT
      evt_block_time AS block_time
     , evt_block_number AS block_number
     , tokenId AS nft_token_id
     , 'Private Sale' AS trade_category
-    , case when (sellerRev = 0 and cast(creatorFee as decimal(38)) > 0 ) then 'primary' else 'secondary' end as trade_type
+    , case when (sellerRev = UINT256 '0' and creatorFee > UINT256 '0') then 'primary' else 'secondary' end as trade_type
     , seller
     , buyer
     , creatorFee+protocolFee+sellerRev AS price_raw
     , contract_address AS project_contract_address
     , nftContract AS nft_contract_address
     , evt_tx_hash AS tx_hash
     , protocolFee AS platform_fee_amount_raw
-    , case when (sellerRev = 0 and cast(creatorFee as decimal(38)) > 0 ) then 0 else creatorFee end AS royalty_fee_amount_raw
+    , case when (sellerRev = UINT256 '0' and creatorFee > UINT256 '0') then UINT256 '0' else creatorFee end AS royalty_fee_amount_raw
     , evt_index as sub_tx_trade_id
     FROM {{ source('foundation_ethereum','market_evt_PrivateSaleFinalized') }} f
      {% if is_incremental() %} -- this filter will only be applied on an incremental run
-     WHERE f.evt_block_time >= date_trunc("day", now() - interval '1 week')
+     WHERE f.evt_block_time >= date_trunc('day', now() - interval '7' day)
      {% else %}
-     WHERE f.evt_block_time >= '{{project_start_date}}'
+     WHERE f.evt_block_time >= TIMESTAMP '{{project_start_date}}'
      {% endif %}
     )
 
@@ -106,19 +107,19 @@ SELECT
 , t.block_time
 , t.block_number
 , t.nft_token_id
-, 1 AS nft_amount
+, UINT256 '1' AS nft_amount
 , t.trade_category
 , t.trade_type
 , t.seller
 , t.buyer
-, cast(t.price_raw as decimal(38)) as price_raw
-, '{{ var("ETH_ERC20_ADDRESS") }}' AS currency_contract -- all trades are in ETH
+, cast(t.price_raw as UINT256) as price_raw
+, {{var("ETH_ERC20_ADDRESS")}} AS currency_contract -- all trades are in ETH
 , t.project_contract_address
 , t.nft_contract_address
 , t.tx_hash
-, cast(t.platform_fee_amount_raw as decimal(38)) as platform_fee_amount_raw
-, cast(t.royalty_fee_amount_raw as decimal(38)) as royalty_fee_amount_raw
-, cast(NULL as string) AS royalty_fee_address
-, cast(NULL as string) AS platform_fee_address
+, cast(t.platform_fee_amount_raw as UINT256) as platform_fee_amount_raw
+, cast(t.royalty_fee_amount_raw as UINT256) as royalty_fee_amount_raw
+, cast(NULL as varbinary) AS royalty_fee_address
+, cast(NULL as varbinary) AS platform_fee_address
 , t.sub_tx_trade_id
 FROM all_foundation_trades t

models/_sector/nft/trades/ethereum/platforms/foundation_ethereum_base_trades_legacy.sql

@@ -0,0 +1,124 @@
+{{ config(
+    schema = 'foundation_ethereum',
+    alias = alias('base_trades', legacy_model=True),
+    partition_by = ['block_date'],
+    materialized = 'incremental',
+    file_format = 'delta',
+    incremental_strategy = 'merge',
+    unique_key = ['block_number','tx_hash','sub_tx_trade_id'],
+    )
+}}
+{% set project_start_date='2021-2-04' %}
+
+WITH all_foundation_trades AS (
+    SELECT
+     f.evt_block_time AS block_time
+    , f.evt_block_number AS block_number
+    , c.tokenId AS nft_token_id
+    , 'Auction Settled' AS trade_category
+    , case when (f.sellerRev = 0 and cast(f.creatorRev as decimal(38)) > 0 ) then 'primary' else 'secondary' end as trade_type
+    , f.seller
+    , f.bidder AS buyer
+    , f.creatorRev+f.totalFees+f.sellerRev AS price_raw
+    , f.contract_address AS project_contract_address
+    , c.nftContract AS nft_contract_address
+    , f.evt_tx_hash AS tx_hash
+    , f.totalFees AS platform_fee_amount_raw
+    , case when (f.sellerRev = 0 and cast(f.creatorRev as decimal(38)) > 0 ) then 0 else f.creatorRev end AS royalty_fee_amount_raw
+    , f.evt_index as sub_tx_trade_id
+    FROM {{ source('foundation_ethereum','market_evt_ReserveAuctionFinalized') }} f
+    INNER JOIN {{ source('foundation_ethereum','market_evt_ReserveAuctionCreated') }} c ON c.auctionId=f.auctionId AND c.evt_block_time<=f.evt_block_time
+     {% if is_incremental() %} -- this filter will only be applied on an incremental run
+     WHERE f.evt_block_time >= date_trunc("day", now() - interval '1 week')
+     {% else %}
+     WHERE f.evt_block_time >= '{{project_start_date}}'
+     {% endif %}
+    UNION ALL
+    SELECT
+     evt_block_time AS block_time
+    , evt_block_number AS block_number
+    , tokenId AS nft_token_id
+    , 'Buy' AS trade_category
+    , case when (sellerRev = 0 and cast(creatorRev as decimal(38)) > 0 ) then 'primary' else 'secondary' end as trade_type
+    , seller
+    , buyer
+    , creatorRev+totalFees+sellerRev AS price_raw
+    , contract_address AS project_contract_address
+    , nftContract AS nft_contract_address
+    , evt_tx_hash AS tx_hash
+    , totalFees AS platform_fee_amount_raw
+    , case when (sellerRev = 0 and cast(creatorRev as decimal(38)) > 0 ) then 0 else creatorRev end AS royalty_fee_amount_raw
+    , evt_index as sub_tx_trade_id
+    FROM {{ source('foundation_ethereum','market_evt_BuyPriceAccepted') }} f
+     {% if is_incremental() %} -- this filter will only be applied on an incremental run
+     WHERE f.evt_block_time >= date_trunc("day", now() - interval '1 week')
+     {% else %}
+     WHERE f.evt_block_time >= '{{project_start_date}}'
+     {% endif %}
+    UNION ALL
+    SELECT
+     evt_block_time AS block_time
+    , evt_block_number AS block_number
+    , tokenId AS nft_token_id
+    , 'Sell' AS trade_category
+    , case when (sellerRev = 0 and cast(creatorRev as decimal(38)) > 0 ) then 'primary' else 'secondary' end as trade_type
+    , seller
+    , buyer
+    , creatorRev+totalFees+sellerRev AS price_raw
+    , contract_address AS project_contract_address
+    , nftContract AS nft_contract_address
+    , evt_tx_hash AS tx_hash
+    , totalFees AS platform_fee_amount_raw
+    , case when (sellerRev = 0 and cast(creatorRev as decimal(38)) > 0 ) then 0 else creatorRev end AS royalty_fee_amount_raw
+    , evt_index as sub_tx_trade_id
+    FROM {{ source('foundation_ethereum','market_evt_OfferAccepted') }} f
+     {% if is_incremental() %} -- this filter will only be applied on an incremental run
+     WHERE f.evt_block_time >= date_trunc("day", now() - interval '1 week')
+     {% else %}
+     WHERE f.evt_block_time >= '{{project_start_date}}'
+     {% endif %}
+    UNION ALL
+    SELECT
+     evt_block_time AS block_time
+    , evt_block_number AS block_number
+    , tokenId AS nft_token_id
+    , 'Private Sale' AS trade_category
+    , case when (sellerRev = 0 and cast(creatorFee as decimal(38)) > 0 ) then 'primary' else 'secondary' end as trade_type
+    , seller
+    , buyer
+    , creatorFee+protocolFee+sellerRev AS price_raw
+    , contract_address AS project_contract_address
+    , nftContract AS nft_contract_address
+    , evt_tx_hash AS tx_hash
+    , protocolFee AS platform_fee_amount_raw
+    , case when (sellerRev = 0 and cast(creatorFee as decimal(38)) > 0 ) then 0 else creatorFee end AS royalty_fee_amount_raw
+    , evt_index as sub_tx_trade_id
+    FROM {{ source('foundation_ethereum','market_evt_PrivateSaleFinalized') }} f
+     {% if is_incremental() %} -- this filter will only be applied on an incremental run
+     WHERE f.evt_block_time >= date_trunc("day", now() - interval '1 week')
+     {% else %}
+     WHERE f.evt_block_time >= '{{project_start_date}}'
+     {% endif %}
+    )
+
+SELECT
+ date_trunc('day', t.block_time) AS block_date
+, t.block_time
+, t.block_number
+, t.nft_token_id
+, 1 AS nft_amount
+, t.trade_category
+, t.trade_type
+, t.seller
+, t.buyer
+, cast(t.price_raw as decimal(38)) as price_raw
+, '{{ var("ETH_ERC20_ADDRESS") }}' AS currency_contract -- all trades are in ETH
+, t.project_contract_address
+, t.nft_contract_address
+, t.tx_hash
+, cast(t.platform_fee_amount_raw as decimal(38)) as platform_fee_amount_raw
+, cast(t.royalty_fee_amount_raw as decimal(38)) as royalty_fee_amount_raw
+, cast(NULL as string) AS royalty_fee_address
+, cast(NULL as string) AS platform_fee_address
+, t.sub_tx_trade_id
+FROM all_foundation_trades t

models/_sector/nft/trades/nft_ethereum_trades_beta.sql

@@ -12,7 +12,7 @@
 {% set base_models = [
      ('archipelago',    'v1',   ref('archipelago_ethereum_base_trades'))
     ,('superrare',    'v1',   ref('superrare_ethereum_base_trades'))
-    ,('foundation',    'v1',   ref('foundation_ethereum_base_trades'))
+    ,('foundation',    'v1',   ref('foundation_ethereum_base_trades_legacy'))
     ,('blur',    'v1',   ref('blur_ethereum_base_trades'))
     ,('element',    'v1',   ref('element_ethereum_base_trades'))
     ,('x2y2',    'v1',   ref('x2y2_ethereum_base_trades'))

seeds/_sector/nft/_schema.yml

@@ -10,10 +10,16 @@ seeds:
 
   - name: foundation_ethereum_base_trades_seed
     config:
+      tags: dunesql
       column_types:
-        price_raw: decimal(38)
-        platform_fee_amount_raw: decimal(38)
-        royalty_fee_amount_raw: decimal(38)
+        tx_hash: varbinary
+        buyer: varbinary
+        seller: varbinary
+        nft_contract_address: varbinary
+        price_raw: uint256
+        platform_fee_amount_raw: uint256
+        royalty_fee_amount_raw: uint256
+        nft_token_id: uint256
 
   - name: superrare_ethereum_base_trades_seed
     config: