Rename {Create,Lazy}TokenStream as {To,Lazy}AttrTokenStream.

`To` is better than `Create` for indicating that this is a non-consuming
conversion, rather than creating something out of nothing.

And the addition of `Attr` is because the current names makes them sound
like they relate to `TokenStream`, but really they relate to
`AttrTokenStream`.

Author: nnethercote

compiler/rustc_ast/src/ast.rs

@@ -24,7 +24,7 @@ pub use UnsafeSource::*;
 
 use crate::ptr::P;
 use crate::token::{self, CommentKind, Delimiter};
-use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream};
+use crate::tokenstream::{DelimSpan, LazyAttrTokenStream, TokenStream};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::sync::Lrc;
@@ -92,7 +92,7 @@ pub struct Path {
     /// The segments in the path: the things separated by `::`.
     /// Global paths begin with `kw::PathRoot`.
     pub segments: Vec<PathSegment>,
-    pub tokens: Option<LazyTokenStream>,
+    pub tokens: Option<LazyAttrTokenStream>,
 }
 
 impl PartialEq<Symbol> for Path {
@@ -564,7 +564,7 @@ pub struct Block {
     /// Distinguishes between `unsafe { ... }` and `{ ... }`.
     pub rules: BlockCheckMode,
     pub span: Span,
-    pub tokens: Option<LazyTokenStream>,
+    pub tokens: Option<LazyAttrTokenStream>,
     /// The following *isn't* a parse error, but will cause multiple errors in following stages.
     /// ```compile_fail
     /// let x = {
@@ -583,7 +583,7 @@ pub struct Pat {
     pub id: NodeId,
     pub kind: PatKind,
     pub span: Span,
-    pub tokens: Option<LazyTokenStream>,
+    pub tokens: Option<LazyAttrTokenStream>,
 }
 
 impl Pat {
@@ -967,8 +967,8 @@ impl Stmt {
     /// a trailing semicolon.
     ///
     /// This only modifies the parsed AST struct, not the attached
-    /// `LazyTokenStream`. The parser is responsible for calling
-    /// `CreateTokenStream::add_trailing_semi` when there is actually
+    /// `LazyAttrTokenStream`. The parser is responsible for calling
+    /// `ToAttrTokenStream::add_trailing_semi` when there is actually
     /// a semicolon in the tokenstream.
     pub fn add_trailing_semicolon(mut self) -> Self {
         self.kind = match self.kind {
@@ -1014,7 +1014,7 @@ pub struct MacCallStmt {
     pub mac: P<MacCall>,
     pub style: MacStmtStyle,
     pub attrs: AttrVec,
-    pub tokens: Option<LazyTokenStream>,
+    pub tokens: Option<LazyAttrTokenStream>,
 }
 
 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug)]
@@ -1039,7 +1039,7 @@ pub struct Local {
     pub kind: LocalKind,
     pub span: Span,
     pub attrs: AttrVec,
-    pub tokens: Option<LazyTokenStream>,
+    pub tokens: Option<LazyAttrTokenStream>,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
@@ -1138,7 +1138,7 @@ pub struct Expr {
     pub kind: ExprKind,
     pub span: Span,
     pub attrs: AttrVec,
-    pub tokens: Option<LazyTokenStream>,
+    pub tokens: Option<LazyAttrTokenStream>,
 }
 
 impl Expr {
@@ -1997,7 +1997,7 @@ pub struct Ty {
     pub id: NodeId,
     pub kind: TyKind,
     pub span: Span,
-    pub tokens: Option<LazyTokenStream>,
+    pub tokens: Option<LazyAttrTokenStream>,
 }
 
 impl Clone for Ty {
@@ -2562,7 +2562,7 @@ impl<D: Decoder> Decodable<D> for AttrId {
 pub struct AttrItem {
     pub path: Path,
     pub args: MacArgs,
-    pub tokens: Option<LazyTokenStream>,
+    pub tokens: Option<LazyAttrTokenStream>,
 }
 
 /// A list of attributes.
@@ -2582,7 +2582,7 @@ pub struct Attribute {
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct NormalAttr {
     pub item: AttrItem,
-    pub tokens: Option<LazyTokenStream>,
+    pub tokens: Option<LazyAttrTokenStream>,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
@@ -2633,7 +2633,7 @@ impl PolyTraitRef {
 pub struct Visibility {
     pub kind: VisibilityKind,
     pub span: Span,
-    pub tokens: Option<LazyTokenStream>,
+    pub tokens: Option<LazyAttrTokenStream>,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
@@ -2719,7 +2719,7 @@ pub struct Item<K = ItemKind> {
     ///
     /// Note that the tokens here do not include the outer attributes, but will
     /// include inner attributes.
-    pub tokens: Option<LazyTokenStream>,
+    pub tokens: Option<LazyAttrTokenStream>,
 }
 
 impl Item {

compiler/rustc_ast/src/ast_traits.rs

@@ -4,7 +4,7 @@
 
 use crate::ptr::P;
 use crate::token::Nonterminal;
-use crate::tokenstream::LazyTokenStream;
+use crate::tokenstream::LazyAttrTokenStream;
 use crate::{Arm, Crate, ExprField, FieldDef, GenericParam, Param, PatField, Variant};
 use crate::{AssocItem, Expr, ForeignItem, Item, NodeId};
 use crate::{AttrItem, AttrKind, Block, Pat, Path, Ty, Visibility};
@@ -124,18 +124,18 @@ impl HasSpan for AttrItem {
 
 /// A trait for AST nodes having (or not having) collected tokens.
 pub trait HasTokens {
-    fn tokens(&self) -> Option<&LazyTokenStream>;
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>>;
+    fn tokens(&self) -> Option<&LazyAttrTokenStream>;
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>>;
 }
 
 macro_rules! impl_has_tokens {
     ($($T:ty),+ $(,)?) => {
         $(
             impl HasTokens for $T {
-                fn tokens(&self) -> Option<&LazyTokenStream> {
+                fn tokens(&self) -> Option<&LazyAttrTokenStream> {
                     self.tokens.as_ref()
                 }
-                fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+                fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
                     Some(&mut self.tokens)
                 }
             }
@@ -147,10 +147,10 @@ macro_rules! impl_has_tokens_none {
     ($($T:ty),+ $(,)?) => {
         $(
             impl HasTokens for $T {
-                fn tokens(&self) -> Option<&LazyTokenStream> {
+                fn tokens(&self) -> Option<&LazyAttrTokenStream> {
                     None
                 }
-                fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+                fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
                     None
                 }
             }
@@ -162,25 +162,25 @@ impl_has_tokens!(AssocItem, AttrItem, Block, Expr, ForeignItem, Item, Pat, Path,
 impl_has_tokens_none!(Arm, ExprField, FieldDef, GenericParam, Param, PatField, Variant);
 
 impl<T: AstDeref<Target: HasTokens>> HasTokens for T {
-    fn tokens(&self) -> Option<&LazyTokenStream> {
+    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
         self.ast_deref().tokens()
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
         self.ast_deref_mut().tokens_mut()
     }
 }
 
 impl<T: HasTokens> HasTokens for Option<T> {
-    fn tokens(&self) -> Option<&LazyTokenStream> {
+    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
         self.as_ref().and_then(|inner| inner.tokens())
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
         self.as_mut().and_then(|inner| inner.tokens_mut())
     }
 }
 
 impl HasTokens for StmtKind {
-    fn tokens(&self) -> Option<&LazyTokenStream> {
+    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
         match self {
             StmtKind::Local(local) => local.tokens.as_ref(),
             StmtKind::Item(item) => item.tokens(),
@@ -189,7 +189,7 @@ impl HasTokens for StmtKind {
             StmtKind::MacCall(mac) => mac.tokens.as_ref(),
         }
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
         match self {
             StmtKind::Local(local) => Some(&mut local.tokens),
             StmtKind::Item(item) => item.tokens_mut(),
@@ -201,24 +201,24 @@ impl HasTokens for StmtKind {
 }
 
 impl HasTokens for Stmt {
-    fn tokens(&self) -> Option<&LazyTokenStream> {
+    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
         self.kind.tokens()
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
         self.kind.tokens_mut()
     }
 }
 
 impl HasTokens for Attribute {
-    fn tokens(&self) -> Option<&LazyTokenStream> {
+    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
         match &self.kind {
             AttrKind::Normal(normal) => normal.tokens.as_ref(),
             kind @ AttrKind::DocComment(..) => {
                 panic!("Called tokens on doc comment attr {:?}", kind)
             }
         }
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
         Some(match &mut self.kind {
             AttrKind::Normal(normal) => &mut normal.tokens,
             kind @ AttrKind::DocComment(..) => {
@@ -229,7 +229,7 @@ impl HasTokens for Attribute {
 }
 
 impl HasTokens for Nonterminal {
-    fn tokens(&self) -> Option<&LazyTokenStream> {
+    fn tokens(&self) -> Option<&LazyAttrTokenStream> {
         match self {
             Nonterminal::NtItem(item) => item.tokens(),
             Nonterminal::NtStmt(stmt) => stmt.tokens(),
@@ -243,7 +243,7 @@ impl HasTokens for Nonterminal {
             Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
         }
     }
-    fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
+    fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
         match self {
             Nonterminal::NtItem(item) => item.tokens_mut(),
             Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),

compiler/rustc_ast/src/attr/mod.rs

@@ -8,7 +8,7 @@ use crate::ast::{Path, PathSegment};
 use crate::ptr::P;
 use crate::token::{self, CommentKind, Delimiter, Token};
 use crate::tokenstream::{DelimSpan, Spacing, TokenTree};
-use crate::tokenstream::{LazyTokenStream, TokenStream};
+use crate::tokenstream::{LazyAttrTokenStream, TokenStream};
 use crate::util::comments;
 
 use rustc_index::bit_set::GrowableBitSet;
@@ -301,7 +301,7 @@ impl Attribute {
                 .tokens
                 .as_ref()
                 .unwrap_or_else(|| panic!("attribute is missing tokens: {:?}", self))
-                .create_token_stream()
+                .to_attr_token_stream()
                 .to_tokenstream(),
             AttrKind::DocComment(comment_kind, data) => TokenStream::new(vec![TokenTree::Token(
                 Token::new(token::DocComment(comment_kind, self.style, data), self.span),
@@ -353,7 +353,7 @@ pub fn mk_attr(style: AttrStyle, path: Path, args: MacArgs, span: Span) -> Attri
 
 pub fn mk_attr_from_item(
     item: AttrItem,
-    tokens: Option<LazyTokenStream>,
+    tokens: Option<LazyAttrTokenStream>,
     style: AttrStyle,
     span: Span,
 ) -> Attribute {

compiler/rustc_ast/src/mut_visit.rs

@@ -697,17 +697,20 @@ pub fn visit_attr_tts<T: MutVisitor>(AttrTokenStream(tts): &mut AttrTokenStream,
     }
 }
 
-pub fn visit_lazy_tts_opt_mut<T: MutVisitor>(lazy_tts: Option<&mut LazyTokenStream>, vis: &mut T) {
+pub fn visit_lazy_tts_opt_mut<T: MutVisitor>(
+    lazy_tts: Option<&mut LazyAttrTokenStream>,
+    vis: &mut T,
+) {
     if T::VISIT_TOKENS {
         if let Some(lazy_tts) = lazy_tts {
-            let mut tts = lazy_tts.create_token_stream();
+            let mut tts = lazy_tts.to_attr_token_stream();
             visit_attr_tts(&mut tts, vis);
-            *lazy_tts = LazyTokenStream::new(tts);
+            *lazy_tts = LazyAttrTokenStream::new(tts);
         }
     }
 }
 
-pub fn visit_lazy_tts<T: MutVisitor>(lazy_tts: &mut Option<LazyTokenStream>, vis: &mut T) {
+pub fn visit_lazy_tts<T: MutVisitor>(lazy_tts: &mut Option<LazyAttrTokenStream>, vis: &mut T) {
     visit_lazy_tts_opt_mut(lazy_tts.as_mut(), vis);
 }