|  | 
| 52 | 52 | 
 | 
| 53 | 53 | mod by_move_body; | 
| 54 | 54 | mod drop; | 
| 55 |  | -use std::{iter, ops}; | 
|  | 55 | +use std::ops; | 
| 56 | 56 | 
 | 
| 57 | 57 | pub(super) use by_move_body::coroutine_by_move_body_def_id; | 
| 58 | 58 | use drop::{ | 
| 59 | 59 |     cleanup_async_drops, create_coroutine_drop_shim, create_coroutine_drop_shim_async, | 
| 60 | 60 |     create_coroutine_drop_shim_proxy_async, elaborate_coroutine_drops, expand_async_drops, | 
| 61 | 61 |     has_expandable_async_drops, insert_clean_drop, | 
| 62 | 62 | }; | 
|  | 63 | +use itertools::izip; | 
| 63 | 64 | use rustc_abi::{FieldIdx, VariantIdx}; | 
| 64 | 65 | use rustc_data_structures::fx::FxHashSet; | 
| 65 | 66 | use rustc_errors::pluralize; | 
| @@ -752,53 +753,53 @@ fn locals_live_across_suspend_points<'tcx>( | 
| 752 | 753 |     let mut live_locals_at_any_suspension_point = DenseBitSet::new_empty(body.local_decls.len()); | 
| 753 | 754 | 
 | 
| 754 | 755 |     for (block, data) in body.basic_blocks.iter_enumerated() { | 
| 755 |  | -        if let TerminatorKind::Yield { .. } = data.terminator().kind { | 
| 756 |  | -            let loc = Location { block, statement_index: data.statements.len() }; | 
| 757 |  | - | 
| 758 |  | -            liveness.seek_to_block_end(block); | 
| 759 |  | -            let mut live_locals = liveness.get().clone(); | 
| 760 |  | - | 
| 761 |  | -            if !movable { | 
| 762 |  | -                // The `liveness` variable contains the liveness of MIR locals ignoring borrows. | 
| 763 |  | -                // This is correct for movable coroutines since borrows cannot live across | 
| 764 |  | -                // suspension points. However for immovable coroutines we need to account for | 
| 765 |  | -                // borrows, so we conservatively assume that all borrowed locals are live until | 
| 766 |  | -                // we find a StorageDead statement referencing the locals. | 
| 767 |  | -                // To do this we just union our `liveness` result with `borrowed_locals`, which | 
| 768 |  | -                // contains all the locals which has been borrowed before this suspension point. | 
| 769 |  | -                // If a borrow is converted to a raw reference, we must also assume that it lives | 
| 770 |  | -                // forever. Note that the final liveness is still bounded by the storage liveness | 
| 771 |  | -                // of the local, which happens using the `intersect` operation below. | 
| 772 |  | -                borrowed_locals_cursor2.seek_before_primary_effect(loc); | 
| 773 |  | -                live_locals.union(borrowed_locals_cursor2.get()); | 
| 774 |  | -            } | 
|  | 756 | +        let TerminatorKind::Yield { .. } = data.terminator().kind else { continue }; | 
|  | 757 | + | 
|  | 758 | +        let loc = Location { block, statement_index: data.statements.len() }; | 
|  | 759 | + | 
|  | 760 | +        liveness.seek_to_block_end(block); | 
|  | 761 | +        let mut live_locals = liveness.get().clone(); | 
|  | 762 | + | 
|  | 763 | +        if !movable { | 
|  | 764 | +            // The `liveness` variable contains the liveness of MIR locals ignoring borrows. | 
|  | 765 | +            // This is correct for movable coroutines since borrows cannot live across | 
|  | 766 | +            // suspension points. However for immovable coroutines we need to account for | 
|  | 767 | +            // borrows, so we conservatively assume that all borrowed locals are live until | 
|  | 768 | +            // we find a StorageDead statement referencing the locals. | 
|  | 769 | +            // To do this we just union our `liveness` result with `borrowed_locals`, which | 
|  | 770 | +            // contains all the locals which has been borrowed before this suspension point. | 
|  | 771 | +            // If a borrow is converted to a raw reference, we must also assume that it lives | 
|  | 772 | +            // forever. Note that the final liveness is still bounded by the storage liveness | 
|  | 773 | +            // of the local, which happens using the `intersect` operation below. | 
|  | 774 | +            borrowed_locals_cursor2.seek_before_primary_effect(loc); | 
|  | 775 | +            live_locals.union(borrowed_locals_cursor2.get()); | 
|  | 776 | +        } | 
| 775 | 777 | 
 | 
| 776 |  | -            // Store the storage liveness for later use so we can restore the state | 
| 777 |  | -            // after a suspension point | 
| 778 |  | -            storage_live.seek_before_primary_effect(loc); | 
| 779 |  | -            storage_liveness_map[block] = Some(storage_live.get().clone()); | 
|  | 778 | +        // Store the storage liveness for later use so we can restore the state | 
|  | 779 | +        // after a suspension point | 
|  | 780 | +        storage_live.seek_before_primary_effect(loc); | 
|  | 781 | +        storage_liveness_map[block] = Some(storage_live.get().clone()); | 
| 780 | 782 | 
 | 
| 781 |  | -            // Locals live are live at this point only if they are used across | 
| 782 |  | -            // suspension points (the `liveness` variable) | 
| 783 |  | -            // and their storage is required (the `storage_required` variable) | 
| 784 |  | -            requires_storage_cursor.seek_before_primary_effect(loc); | 
| 785 |  | -            live_locals.intersect(requires_storage_cursor.get()); | 
|  | 783 | +        // Locals live are live at this point only if they are used across | 
|  | 784 | +        // suspension points (the `liveness` variable) | 
|  | 785 | +        // and their storage is required (the `storage_required` variable) | 
|  | 786 | +        requires_storage_cursor.seek_before_primary_effect(loc); | 
|  | 787 | +        live_locals.intersect(requires_storage_cursor.get()); | 
| 786 | 788 | 
 | 
| 787 |  | -            // The coroutine argument is ignored. | 
| 788 |  | -            live_locals.remove(SELF_ARG); | 
|  | 789 | +        // The coroutine argument is ignored. | 
|  | 790 | +        live_locals.remove(SELF_ARG); | 
| 789 | 791 | 
 | 
| 790 |  | -            debug!("loc = {:?}, live_locals = {:?}", loc, live_locals); | 
|  | 792 | +        debug!(?loc, ?live_locals); | 
| 791 | 793 | 
 | 
| 792 |  | -            // Add the locals live at this suspension point to the set of locals which live across | 
| 793 |  | -            // any suspension points | 
| 794 |  | -            live_locals_at_any_suspension_point.union(&live_locals); | 
|  | 794 | +        // Add the locals live at this suspension point to the set of locals which live across | 
|  | 795 | +        // any suspension points | 
|  | 796 | +        live_locals_at_any_suspension_point.union(&live_locals); | 
| 795 | 797 | 
 | 
| 796 |  | -            live_locals_at_suspension_points.push(live_locals); | 
| 797 |  | -            source_info_at_suspension_points.push(data.terminator().source_info); | 
| 798 |  | -        } | 
|  | 798 | +        live_locals_at_suspension_points.push(live_locals); | 
|  | 799 | +        source_info_at_suspension_points.push(data.terminator().source_info); | 
| 799 | 800 |     } | 
| 800 | 801 | 
 | 
| 801 |  | -    debug!("live_locals_anywhere = {:?}", live_locals_at_any_suspension_point); | 
|  | 802 | +    debug!(?live_locals_at_any_suspension_point); | 
| 802 | 803 |     let saved_locals = CoroutineSavedLocals(live_locals_at_any_suspension_point); | 
| 803 | 804 | 
 | 
| 804 | 805 |     // Renumber our liveness_map bitsets to include only the locals we are | 
| @@ -999,8 +1000,8 @@ fn compute_layout<'tcx>( | 
| 999 | 1000 |     } = liveness; | 
| 1000 | 1001 | 
 | 
| 1001 | 1002 |     // Gather live local types and their indices. | 
| 1002 |  | -    let mut locals = IndexVec::<CoroutineSavedLocal, _>::new(); | 
| 1003 |  | -    let mut tys = IndexVec::<CoroutineSavedLocal, _>::new(); | 
|  | 1003 | +    let mut locals = IndexVec::<CoroutineSavedLocal, _>::with_capacity(saved_locals.domain_size()); | 
|  | 1004 | +    let mut tys = IndexVec::<CoroutineSavedLocal, _>::with_capacity(saved_locals.domain_size()); | 
| 1004 | 1005 |     for (saved_local, local) in saved_locals.iter_enumerated() { | 
| 1005 | 1006 |         debug!("coroutine saved local {:?} => {:?}", saved_local, local); | 
| 1006 | 1007 | 
 | 
| @@ -1034,38 +1035,39 @@ fn compute_layout<'tcx>( | 
| 1034 | 1035 |     // In debuginfo, these will correspond to the beginning (UNRESUMED) or end | 
| 1035 | 1036 |     // (RETURNED, POISONED) of the function. | 
| 1036 | 1037 |     let body_span = body.source_scopes[OUTERMOST_SOURCE_SCOPE].span; | 
| 1037 |  | -    let mut variant_source_info: IndexVec<VariantIdx, SourceInfo> = [ | 
|  | 1038 | +    let mut variant_source_info: IndexVec<VariantIdx, SourceInfo> = IndexVec::with_capacity( | 
|  | 1039 | +        CoroutineArgs::RESERVED_VARIANTS + live_locals_at_suspension_points.len(), | 
|  | 1040 | +    ); | 
|  | 1041 | +    variant_source_info.extend([ | 
| 1038 | 1042 |         SourceInfo::outermost(body_span.shrink_to_lo()), | 
| 1039 | 1043 |         SourceInfo::outermost(body_span.shrink_to_hi()), | 
| 1040 | 1044 |         SourceInfo::outermost(body_span.shrink_to_hi()), | 
| 1041 |  | -    ] | 
| 1042 |  | -    .iter() | 
| 1043 |  | -    .copied() | 
| 1044 |  | -    .collect(); | 
|  | 1045 | +    ]); | 
| 1045 | 1046 | 
 | 
| 1046 | 1047 |     // Build the coroutine variant field list. | 
| 1047 | 1048 |     // Create a map from local indices to coroutine struct indices. | 
| 1048 |  | -    let mut variant_fields: IndexVec<VariantIdx, IndexVec<FieldIdx, CoroutineSavedLocal>> = | 
| 1049 |  | -        iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); | 
|  | 1049 | +    let mut variant_fields: IndexVec<VariantIdx, _> = IndexVec::from_elem_n( | 
|  | 1050 | +        IndexVec::new(), | 
|  | 1051 | +        CoroutineArgs::RESERVED_VARIANTS + live_locals_at_suspension_points.len(), | 
|  | 1052 | +    ); | 
| 1050 | 1053 |     let mut remap = IndexVec::from_elem_n(None, saved_locals.domain_size()); | 
| 1051 |  | -    for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter().enumerate() { | 
| 1052 |  | -        let variant_index = | 
| 1053 |  | -            VariantIdx::from(CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx); | 
| 1054 |  | -        let mut fields = IndexVec::new(); | 
| 1055 |  | -        for (idx, saved_local) in live_locals.iter().enumerate() { | 
| 1056 |  | -            fields.push(saved_local); | 
|  | 1054 | +    for (live_locals, &source_info_at_suspension_point, (variant_index, fields)) in izip!( | 
|  | 1055 | +        &live_locals_at_suspension_points, | 
|  | 1056 | +        &source_info_at_suspension_points, | 
|  | 1057 | +        variant_fields.iter_enumerated_mut().skip(CoroutineArgs::RESERVED_VARIANTS) | 
|  | 1058 | +    ) { | 
|  | 1059 | +        *fields = live_locals.iter().collect(); | 
|  | 1060 | +        for (idx, &saved_local) in fields.iter_enumerated() { | 
| 1057 | 1061 |             // Note that if a field is included in multiple variants, we will | 
| 1058 | 1062 |             // just use the first one here. That's fine; fields do not move | 
| 1059 | 1063 |             // around inside coroutines, so it doesn't matter which variant | 
| 1060 | 1064 |             // index we access them by. | 
| 1061 |  | -            let idx = FieldIdx::from_usize(idx); | 
| 1062 | 1065 |             remap[locals[saved_local]] = Some((tys[saved_local].ty, variant_index, idx)); | 
| 1063 | 1066 |         } | 
| 1064 |  | -        variant_fields.push(fields); | 
| 1065 |  | -        variant_source_info.push(source_info_at_suspension_points[suspension_point_idx]); | 
|  | 1067 | +        variant_source_info.push(source_info_at_suspension_point); | 
| 1066 | 1068 |     } | 
| 1067 |  | -    debug!("coroutine variant_fields = {:?}", variant_fields); | 
| 1068 |  | -    debug!("coroutine storage_conflicts = {:#?}", storage_conflicts); | 
|  | 1069 | +    debug!(?variant_fields); | 
|  | 1070 | +    debug!(?storage_conflicts); | 
| 1069 | 1071 | 
 | 
| 1070 | 1072 |     let mut field_names = IndexVec::from_elem(None, &tys); | 
| 1071 | 1073 |     for var in &body.var_debug_info { | 
|  | 
0 commit comments