Historical block/state roots

setup.py

@@ -201,7 +201,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str]) ->
                 # NOTE: trim whitespace from spec
                 ssz_objects[current_name] = "\n".join(line.rstrip() for line in source.splitlines())
             else:
-                raise Exception("unrecognized python code element")
+                raise Exception("unrecognized python code element: " + source)
         elif isinstance(child, Table):
             for row in child.children:
                 cells = row.children
@@ -831,7 +831,7 @@ def initialize_options(self):
         self.out_dir = 'pyspec_output'
         self.build_targets = """
                 minimal:presets/minimal:configs/minimal.yaml
-                mainnet:presets/mainnet:configs/mainnet.yaml        
+                mainnet:presets/mainnet:configs/mainnet.yaml
         """
 
     def finalize_options(self):

specs/altair/beacon-chain.md

@@ -23,6 +23,7 @@
     - [`BeaconBlockBody`](#beaconblockbody)
     - [`BeaconState`](#beaconstate)
   - [New containers](#new-containers)
+    - [`HistoricalBatchSummary`](#historicalbatchsummary)
     - [`SyncAggregate`](#syncaggregate)
     - [`SyncCommittee`](#synccommittee)
 - [Helper functions](#helper-functions)
@@ -52,6 +53,7 @@
     - [Rewards and penalties](#rewards-and-penalties)
     - [Slashings](#slashings)
     - [Participation flags updates](#participation-flags-updates)
+    - [Historical batches updates](#historical-batches-updates)
     - [Sync committee updates](#sync-committee-updates)
 - [Initialize state for pure Altair testnets and test vectors](#initialize-state-for-pure-altair-testnets-and-test-vectors)
 
@@ -174,7 +176,7 @@ class BeaconState(Container):
     latest_block_header: BeaconBlockHeader
     block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT]
     state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT]
-    historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT]
+    historical_batches: List[HistoricalBatchSummary, HISTORICAL_ROOTS_LIMIT]
     # Eth1
     eth1_data: Eth1Data
     eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH]
@@ -203,6 +205,18 @@ class BeaconState(Container):
 
 ### New containers
 
+#### `HistoricalBatchSummary`
+
+```python
+class HistoricalBatchSummary(Container):
+    """
+    `HistoricalBatchSummary` matches the components of the phase0 HistoricalBatch
+    making the two hash_tree_root-compatible.
+    """
+    block_batch_root: Root
+    state_batch_root: Root
+```
+
 #### `SyncAggregate`
 
 ```python
@@ -598,7 +612,7 @@ def process_epoch(state: BeaconState) -> None:
     process_effective_balance_updates(state)
     process_slashings_reset(state)
     process_randao_mixes_reset(state)
-    process_historical_roots_update(state)
+    process_historical_batches_update(state)
     process_participation_flag_updates(state)  # [New in Altair]
     process_sync_committee_updates(state)  # [New in Altair]
 ```
@@ -687,6 +701,21 @@ def process_participation_flag_updates(state: BeaconState) -> None:
     state.current_epoch_participation = [ParticipationFlags(0b0000_0000) for _ in range(len(state.validators))]
 ```
 
+#### Historical batches updates
+
+*Note*: The function `process_historical_batches_update` replaces `process_historical_roots_update` in phase0.
+
+```python
+def process_historical_batches_update(state: BeaconState) -> None:
+    # Set historical block root accumulator
+    next_epoch = Epoch(get_current_epoch(state) + 1)
+    if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0:
+        historical_batch = HistoricalBatchSummary(
+            block_batch_root=hash_tree_root(state.block_roots),
+            state_batch_root=hash_tree_root(state.state_roots))
+        state.historical_batches.append(historical_batch)
+```
+
 #### Sync committee updates
 
 *Note*: The function `process_sync_committee_updates` is new.

specs/altair/fork.md

@@ -58,9 +58,21 @@ def translate_participation(state: BeaconState, pending_attestations: Sequence[p
             for flag_index in participation_flag_indices:
                 epoch_participation[index] = add_flag(epoch_participation[index], flag_index)
 
+def load_historical_batches(pre: BeaconState) -> List[HistoricalBatchSummary, HISTORICAL_ROOTS_LIMIT]:
+    # Clients need to recalculate the historical batches from genesis and return
+    # them here with the roots of the blocks and states separated. It's assumed
+    # most of these are pre-calculated and the rest are stored on-the-fly by
+    # altair-compatible clients leading up to the fork. All in all, this should
+    # be around 10kb of hashes.
+
+    return List[HistoricalBatchSummary, HISTORICAL_ROOTS_LIMIT]()
 
 def upgrade_to_altair(pre: phase0.BeaconState) -> BeaconState:
     epoch = phase0.get_current_epoch(pre)
+    historical_batches = load_historical_batches(pre)
+
+    # Tree hash should match since HistoricalBatchSummary is an intermediate step in calculating the phase0 historical root entry
+    assert hash_tree_root(historical_batches) == hash_tree_root(pre.historical_roots)
     post = BeaconState(
         # Versioning
         genesis_time=pre.genesis_time,
@@ -75,7 +87,7 @@ def upgrade_to_altair(pre: phase0.BeaconState) -> BeaconState:
         latest_block_header=pre.latest_block_header,
         block_roots=pre.block_roots,
         state_roots=pre.state_roots,
-        historical_roots=pre.historical_roots,
+        historical_batches=historical_batches,
         # Eth1
         eth1_data=pre.eth1_data,
         eth1_data_votes=pre.eth1_data_votes,