Rollup merge of rust-lang#65641 - nnethercote:derive-TokenStream-Encodable-Decodable, r=petrochenkov

Derive `Rustc{En,De}codable` for `TokenStream`.

`TokenStream` used to be a complex type, but it is now just a newtype
around a `Lrc<Vec<TreeAndJoint>>`. Currently it uses custom encoding
that discards the `IsJoint` and custom decoding that adds `NonJoint`
back in for every token tree. This requires building intermediate
`Vec<TokenTree>`s.

This commit makes `TokenStream` derive `Rustc{En,De}codable`. This
simplifies the code, and avoids the creation of the intermediate
vectors, saving up to 3% on various benchmarks. It also changes the AST
JSON output in one test.

r? @petrochenkov

Author: JohnTitor

src/libsyntax/tokenstream.rs

@@ -19,7 +19,6 @@ use syntax_pos::{BytePos, Span, DUMMY_SP};
 #[cfg(target_arch = "x86_64")]
 use rustc_data_structures::static_assert_size;
 use rustc_data_structures::sync::Lrc;
-use rustc_serialize::{Decoder, Decodable, Encoder, Encodable};
 use smallvec::{SmallVec, smallvec};
 
 use std::{iter, mem};
@@ -136,7 +135,7 @@ impl TokenTree {
 /// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s
 /// instead of a representation of the abstract syntax tree.
 /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for back-compat.
-#[derive(Clone, Debug, Default)]
+#[derive(Clone, Debug, Default, RustcEncodable, RustcDecodable)]
 pub struct TokenStream(pub Lrc<Vec<TreeAndJoint>>);
 
 pub type TreeAndJoint = (TokenTree, IsJoint);
@@ -145,7 +144,7 @@ pub type TreeAndJoint = (TokenTree, IsJoint);
 #[cfg(target_arch = "x86_64")]
 static_assert_size!(TokenStream, 8);
 
-#[derive(Clone, Copy, Debug, PartialEq)]
+#[derive(Clone, Copy, Debug, PartialEq, RustcEncodable, RustcDecodable)]
 pub enum IsJoint {
     Joint,
     NonJoint
@@ -460,18 +459,6 @@ impl Cursor {
     }
 }
 
-impl Encodable for TokenStream {
-    fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), E::Error> {
-        self.trees().collect::<Vec<_>>().encode(encoder)
-    }
-}
-
-impl Decodable for TokenStream {
-    fn decode<D: Decoder>(decoder: &mut D) -> Result<TokenStream, D::Error> {
-        Vec::<TokenTree>::decode(decoder).map(|vec| vec.into_iter().collect())
-    }
-}
-
 #[derive(Debug, Copy, Clone, PartialEq, RustcEncodable, RustcDecodable)]
 pub struct DelimSpan {
     pub open: Span,

src/test/ui/ast-json/ast-json-output.stdout

@@ -1 +1 @@
-{"module":{"inner":{"lo":0,"hi":0},"items":[{"ident":{"name":"core","span":{"lo":0,"hi":0}},"attrs":[],"id":0,"kind":{"variant":"ExternCrate","fields":[null]},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"span":{"lo":0,"hi":0},"tokens":[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["extern",false]},"span":{"lo":0,"hi":0}}]},{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate",false]},"span":{"lo":0,"hi":0}}]},{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["core",false]},"span":{"lo":0,"hi":0}}]},{"variant":"Token","fields":[{"kind":"Semi","span":{"lo":0,"hi":0}}]}]}],"inline":true},"attrs":[],"span":{"lo":0,"hi":0}}
+{"module":{"inner":{"lo":0,"hi":0},"items":[{"ident":{"name":"core","span":{"lo":0,"hi":0}},"attrs":[],"id":0,"kind":{"variant":"ExternCrate","fields":[null]},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"span":{"lo":0,"hi":0},"tokens":{"_field0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["extern",false]},"span":{"lo":0,"hi":0}}]},"NonJoint"],[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate",false]},"span":{"lo":0,"hi":0}}]},"NonJoint"],[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["core",false]},"span":{"lo":0,"hi":0}}]},"NonJoint"],[{"variant":"Token","fields":[{"kind":"Semi","span":{"lo":0,"hi":0}}]},"NonJoint"]]}}],"inline":true},"attrs":[],"span":{"lo":0,"hi":0}}