Auto merge of rust-lang#74846 - Aaron1011:fix/pat-token-capture, r=petrochenkov

bors · bors · commit ff5e0f1dc8c5 · 2020-08-21T02:16:35.000Z
Capture tokens for Pat used in macro_rules! argument This extends PR rust-lang#73293 to handle patterns (Pat). Unlike expressions, patterns do not support custom attributes, so we only need to capture tokens during macro_rules! argument parsing.
diff --git a/src/librustc_ast/ast.rs b/src/librustc_ast/ast.rs
@@ -550,6 +550,7 @@ pub struct Pat {
     pub id: NodeId,
     pub kind: PatKind,
     pub span: Span,
+    pub tokens: Option<TokenStream>,
 }
 
 impl Pat {
@@ -2138,6 +2139,7 @@ impl Param {
                 id: DUMMY_NODE_ID,
                 kind: PatKind::Ident(BindingMode::ByValue(mutbl), eself_ident, None),
                 span,
+                tokens: None,
             }),
             span,
             ty,
diff --git a/src/librustc_ast/mut_visit.rs b/src/librustc_ast/mut_visit.rs
@@ -1053,7 +1053,7 @@ pub fn noop_flat_map_foreign_item<T: MutVisitor>(
 }
 
 pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) {
-    let Pat { id, kind, span } = pat.deref_mut();
+    let Pat { id, kind, span, tokens: _ } = pat.deref_mut();
     vis.visit_id(id);
     match kind {
         PatKind::Wild | PatKind::Rest => {}
diff --git a/src/librustc_expand/base.rs b/src/librustc_expand/base.rs
@@ -551,6 +551,7 @@ impl MacResult for MacEager {
                     id: ast::DUMMY_NODE_ID,
                     span: e.span,
                     kind: PatKind::Lit(e),
+                    tokens: None,
                 }));
             }
         }
@@ -597,7 +598,7 @@ impl DummyResult {
 
     /// A plain dummy pattern.
     pub fn raw_pat(sp: Span) -> ast::Pat {
-        ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp }
+        ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp, tokens: None }
     }
 
     /// A plain dummy type.
diff --git a/src/librustc_expand/build.rs b/src/librustc_expand/build.rs
@@ -392,7 +392,7 @@ impl<'a> ExtCtxt<'a> {
     }
 
     pub fn pat(&self, span: Span, kind: PatKind) -> P<ast::Pat> {
-        P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span })
+        P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span, tokens: None })
     }
     pub fn pat_wild(&self, span: Span) -> P<ast::Pat> {
         self.pat(span, PatKind::Wild)
diff --git a/src/librustc_expand/placeholders.rs b/src/librustc_expand/placeholders.rs
@@ -38,7 +38,8 @@ pub fn placeholder(
         })
     };
     let ty = || P(ast::Ty { id, kind: ast::TyKind::MacCall(mac_placeholder()), span });
-    let pat = || P(ast::Pat { id, kind: ast::PatKind::MacCall(mac_placeholder()), span });
+    let pat =
+        || P(ast::Pat { id, kind: ast::PatKind::MacCall(mac_placeholder()), span, tokens: None });
 
     match kind {
         AstFragmentKind::Expr => AstFragment::Expr(expr_placeholder()),
@@ -85,6 +86,7 @@ pub fn placeholder(
             id,
             span,
             kind: ast::PatKind::MacCall(mac_placeholder()),
+            tokens: None,
         })),
         AstFragmentKind::Ty => {
             AstFragment::Ty(P(ast::Ty { id, span, kind: ast::TyKind::MacCall(mac_placeholder()) }))
diff --git a/src/librustc_parse/lib.rs b/src/librustc_parse/lib.rs
@@ -263,6 +263,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
         Nonterminal::NtItem(ref item) => {
             prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span)
         }
+        Nonterminal::NtPat(ref pat) => pat.tokens.clone(),
         Nonterminal::NtIdent(ident, is_raw) => {
             Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into())
         }
diff --git a/src/librustc_parse/parser/diagnostics.rs b/src/librustc_parse/parser/diagnostics.rs
@@ -26,6 +26,7 @@ pub(super) fn dummy_arg(ident: Ident) -> Param {
         id: ast::DUMMY_NODE_ID,
         kind: PatKind::Ident(BindingMode::ByValue(Mutability::Not), ident, None),
         span: ident.span,
+        tokens: None,
     });
     let ty = Ty { kind: TyKind::Err, span: ident.span, id: ast::DUMMY_NODE_ID };
     Param {
@@ -83,7 +84,12 @@ impl RecoverQPath for Pat {
         self.to_ty()
     }
     fn recovered(qself: Option<QSelf>, path: ast::Path) -> Self {
-        Self { span: path.span, kind: PatKind::Path(qself, path), id: ast::DUMMY_NODE_ID }
+        Self {
+            span: path.span,
+            kind: PatKind::Path(qself, path),
+            id: ast::DUMMY_NODE_ID,
+            tokens: None,
+        }
     }
 }
 
@@ -1526,7 +1532,8 @@ impl<'a> Parser<'a> {
         .emit();
 
         // Pretend the pattern is `_`, to avoid duplicate errors from AST validation.
-        let pat = P(Pat { kind: PatKind::Wild, span: pat.span, id: ast::DUMMY_NODE_ID });
+        let pat =
+            P(Pat { kind: PatKind::Wild, span: pat.span, id: ast::DUMMY_NODE_ID, tokens: None });
         Ok((pat, ty))
     }
 
diff --git a/src/librustc_parse/parser/nonterminal.rs b/src/librustc_parse/parser/nonterminal.rs
@@ -116,7 +116,14 @@ impl<'a> Parser<'a> {
                 Some(s) => token::NtStmt(s),
                 None => return Err(self.struct_span_err(self.token.span, "expected a statement")),
             },
-            NonterminalKind::Pat => token::NtPat(self.parse_pat(None)?),
+            NonterminalKind::Pat => {
+                let (mut pat, tokens) = self.collect_tokens(|this| this.parse_pat(None))?;
+                // We have have eaten an NtPat, which could already have tokens
+                if pat.tokens.is_none() {
+                    pat.tokens = Some(tokens);
+                }
+                token::NtPat(pat)
+            }
             NonterminalKind::Expr => {
                 let (mut expr, tokens) = self.collect_tokens(|this| this.parse_expr())?;
                 // If we captured tokens during parsing (due to outer attributes),
diff --git a/src/librustc_parse/parser/pat.rs b/src/librustc_parse/parser/pat.rs
@@ -1007,6 +1007,6 @@ impl<'a> Parser<'a> {
     }
 
     fn mk_pat(&self, span: Span, kind: PatKind) -> P<Pat> {
-        P(Pat { kind, span, id: ast::DUMMY_NODE_ID })
+        P(Pat { kind, span, id: ast::DUMMY_NODE_ID, tokens: None })
     }
 }
diff --git a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
@@ -164,6 +164,7 @@ fn iter_exprs(depth: usize, f: &mut dyn FnMut(P<Expr>)) {
                     id: DUMMY_NODE_ID,
                     kind: PatKind::Wild,
                     span: DUMMY_SP,
+                    tokens: None,
                 });
                 iter_exprs(depth - 1, &mut |e| g(ExprKind::Let(pat.clone(), e)))
             },
diff --git a/src/test/ui/proc-macro/capture-macro-rules-invoke.rs b/src/test/ui/proc-macro/capture-macro-rules-invoke.rs
@@ -1,12 +1,20 @@
 // aux-build:test-macros.rs
 // check-pass
+// compile-flags: -Z span-debug
+// normalize-stdout-test "#\d+" -> "#CTXT"
 
 extern crate test_macros;
-use test_macros::recollect;
+use test_macros::print_bang;
 
 macro_rules! use_expr {
     ($expr:expr) => {
-        recollect!($expr)
+        print_bang!($expr)
+    }
+}
+
+macro_rules! use_pat {
+    ($pat:pat) => {
+        print_bang!($pat)
     }
 }
 
@@ -17,6 +25,10 @@ impl Foo {
     fn use_self(self) {
         drop(use_expr!(self));
     }
+
+    fn with_pat(use_pat!((a, b)): (u32, u32)) {
+        println!("Args: {} {}", a, b);
+    }
 }
 
 fn main() {}
diff --git a/src/test/ui/proc-macro/capture-macro-rules-invoke.stdout b/src/test/ui/proc-macro/capture-macro-rules-invoke.stdout
@@ -0,0 +1,41 @@
+PRINT-BANG INPUT (DISPLAY): self
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Ident {
+                ident: "self",
+                span: $DIR/capture-macro-rules-invoke.rs:26:24: 26:28 (#CTXT),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:11:21: 11:26 (#CTXT),
+    },
+]
+PRINT-BANG INPUT (DISPLAY): (a, b)
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Group {
+                delimiter: Parenthesis,
+                stream: TokenStream [
+                    Ident {
+                        ident: "a",
+                        span: $DIR/capture-macro-rules-invoke.rs:29:27: 29:28 (#CTXT),
+                    },
+                    Punct {
+                        ch: ',',
+                        spacing: Alone,
+                        span: $DIR/capture-macro-rules-invoke.rs:29:28: 29:29 (#CTXT),
+                    },
+                    Ident {
+                        ident: "b",
+                        span: $DIR/capture-macro-rules-invoke.rs:29:30: 29:31 (#CTXT),
+                    },
+                ],
+                span: $DIR/capture-macro-rules-invoke.rs:29:26: 29:32 (#CTXT),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:17:21: 17:25 (#CTXT),
+    },
+]
diff --git a/src/tools/clippy/clippy_lints/src/unnested_or_patterns.rs b/src/tools/clippy/clippy_lints/src/unnested_or_patterns.rs
@@ -340,6 +340,7 @@ fn take_pat(from: &mut Pat) -> Pat {
         id: DUMMY_NODE_ID,
         kind: Wild,
         span: DUMMY_SP,
+        tokens: None
     };
     mem::replace(from, dummy)
 }

Original file line number	Diff line number	Diff line change
`@@ -1053,7 +1053,7 @@ pub fn noop_flat_map_foreign_item<T: MutVisitor>(`
`1053`	`1053`	`}`
`1054`	`1054`
`1055`	`1055`	`pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) {`
`1056`		`- let Pat { id, kind, span } = pat.deref_mut();`
	`1056`	`+ let Pat { id, kind, span, tokens: _ } = pat.deref_mut();`
`1057`	`1057`	`vis.visit_id(id);`
`1058`	`1058`	`match kind {`
`1059`	`1059`	`PatKind::Wild \| PatKind::Rest => {}`
Original file line number	Diff line number	Diff line change
`@@ -551,6 +551,7 @@ impl MacResult for MacEager {`
`551`	`551`	`id: ast::DUMMY_NODE_ID,`
`552`	`552`	`span: e.span,`
`553`	`553`	`kind: PatKind::Lit(e),`
	`554`	`+ tokens: None,`
`554`	`555`	`}));`
`555`	`556`	`}`
`556`	`557`	`}`
`@@ -597,7 +598,7 @@ impl DummyResult {`
`597`	`598`
`598`	`599`	`/// A plain dummy pattern.`
`599`	`600`	`pub fn raw_pat(sp: Span) -> ast::Pat {`
`600`		`- ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp }`
	`601`	`+ ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Wild, span: sp, tokens: None }`
`601`	`602`	`}`
`602`	`603`
`603`	`604`	`/// A plain dummy type.`
Original file line number	Diff line number	Diff line change
`@@ -392,7 +392,7 @@ impl<'a> ExtCtxt<'a> {`
`392`	`392`	`}`
`393`	`393`
`394`	`394`	`pub fn pat(&self, span: Span, kind: PatKind) -> P<ast::Pat> {`
`395`		`- P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span })`
	`395`	`+ P(ast::Pat { id: ast::DUMMY_NODE_ID, kind, span, tokens: None })`
`396`	`396`	`}`
`397`	`397`	`pub fn pat_wild(&self, span: Span) -> P<ast::Pat> {`
`398`	`398`	`self.pat(span, PatKind::Wild)`
Original file line number	Diff line number	Diff line change
`@@ -263,6 +263,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke`
`263`	`263`	`Nonterminal::NtItem(ref item) => {`
`264`	`264`	`prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span)`
`265`	`265`	`}`
	`266`	`+ Nonterminal::NtPat(ref pat) => pat.tokens.clone(),`
`266`	`267`	`Nonterminal::NtIdent(ident, is_raw) => {`
`267`	`268`	`Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into())`
`268`	`269`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1007,6 +1007,6 @@ impl<'a> Parser<'a> {`
`1007`	`1007`	`}`
`1008`	`1008`
`1009`	`1009`	`fn mk_pat(&self, span: Span, kind: PatKind) -> P<Pat> {`
`1010`		`- P(Pat { kind, span, id: ast::DUMMY_NODE_ID })`
	`1010`	`+ P(Pat { kind, span, id: ast::DUMMY_NODE_ID, tokens: None })`
`1011`	`1011`	`}`
`1012`	`1012`	`}`