Fix partially consumed tokens in macro matchers.

Author: jseyfried

src/libsyntax/ext/tt/macro_parser.rs

@@ -476,24 +476,21 @@ pub fn parse(sess: &ParseSess,
                 }
                 rdr.next_token();
             } else /* bb_eis.len() == 1 */ {
-                let mut rust_parser = Parser::new(sess, cfg.clone(), Box::new(rdr.clone()));
-
-                let mut ei = bb_eis.pop().unwrap();
-                match ei.top_elts.get_tt(ei.idx) {
-                    TokenTree::Token(span, MatchNt(_, ident)) => {
+                rdr.next_tok = {
+                    let mut rust_parser = Parser::new(sess, cfg.clone(), Box::new(&mut rdr));
+                    let mut ei = bb_eis.pop().unwrap();
+                    if let TokenTree::Token(span, MatchNt(_, ident)) = ei.top_elts.get_tt(ei.idx) {
                         let match_cur = ei.match_cur;
                         (&mut ei.matches[match_cur]).push(Rc::new(MatchedNonterminal(
                             parse_nt(&mut rust_parser, span, &ident.name.as_str()))));
                         ei.idx += 1;
                         ei.match_cur += 1;
+                    } else {
+                        unreachable!()
                     }
-                    _ => panic!()
-                }
-                cur_eis.push(ei);
-
-                for _ in 0..rust_parser.tokens_consumed {
-                    let _ = rdr.next_token();
-                }
+                    cur_eis.push(ei);
+                    Some(TokenAndSpan { tok: rust_parser.token, sp: rust_parser.span })
+                };
             }
         }
 

src/libsyntax/ext/tt/transcribe.rs

@@ -48,6 +48,7 @@ pub struct TtReader<'a> {
     /* cached: */
     pub cur_tok: Token,
     pub cur_span: Span,
+    pub next_tok: Option<TokenAndSpan>,
     /// Transform doc comments. Only useful in macro invocations
     pub desugar_doc_comments: bool,
     pub fatal_errs: Vec<DiagnosticBuilder<'a>>,
@@ -100,6 +101,7 @@ pub fn new_tt_reader_with_doc_flag(sp_diag: &Handler,
         /* dummy values, never read: */
         cur_tok: token::Eof,
         cur_span: DUMMY_SP,
+        next_tok: None,
         fatal_errs: Vec::new(),
     };
     tt_next_token(&mut r); /* get cur_tok and cur_span set up */
@@ -178,6 +180,9 @@ fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize {
 /// Return the next token from the TtReader.
 /// EFFECT: advances the reader's token field
 pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
+    if let Some(tok) = r.next_tok.take() {
+        return tok;
+    }
     // FIXME(pcwalton): Bad copy?
     let ret_val = TokenAndSpan {
         tok: r.cur_tok.clone(),

src/libsyntax/parse/lexer/mod.rs

@@ -144,7 +144,7 @@ impl<'a> Reader for StringReader<'a> {
 
 impl<'a> Reader for TtReader<'a> {
     fn is_eof(&self) -> bool {
-        self.cur_tok == token::Eof
+        self.peek().tok == token::Eof
     }
     fn try_next_token(&mut self) -> Result<TokenAndSpan, ()> {
         assert!(self.fatal_errs.is_empty());
@@ -165,10 +165,31 @@ impl<'a> Reader for TtReader<'a> {
         self.fatal_errs.clear();
     }
     fn peek(&self) -> TokenAndSpan {
-        TokenAndSpan {
+        self.next_tok.clone().unwrap_or(TokenAndSpan {
             tok: self.cur_tok.clone(),
             sp: self.cur_span,
-        }
+        })
+    }
+}
+
+impl<'a, 'b> Reader for &'b mut TtReader<'a> {
+    fn is_eof(&self) -> bool {
+        (**self).is_eof()
+    }
+    fn try_next_token(&mut self) -> Result<TokenAndSpan, ()> {
+        (**self).try_next_token()
+    }
+    fn fatal(&self, m: &str) -> FatalError {
+        (**self).fatal(m)
+    }
+    fn err(&self, m: &str) {
+        (**self).err(m)
+    }
+    fn emit_fatal_errors(&mut self) {
+        (**self).emit_fatal_errors()
+    }
+    fn peek(&self) -> TokenAndSpan {
+        (**self).peek()
     }
 }