Rollup merge of rust-lang#37524 - alexcrichton:vendor, r=brson

Vendor all rustbuild dependencies in this repo

This commit vendors all crates.io dependencies into the rust-lang/rust repository using the `cargo-vendor` tool. This is done in an effort to make rustbuild distro-ready by ensuring that our source tarballs are self-contained units which don't need extraneous network downloads.

A new `src/vendor` directory is created with all vendored crates, and Cargo, when using rustbuild, is configured to use this directory. Over time we can deduplicate this directory with the actual src tree (e.g. src/librustc_serialize, src/liblibc, src/libgetopts, ...). For now though that's left to a separate commit.

Author: eddyb

.gitattributes

@@ -7,3 +7,4 @@
 src/etc/pkg/rust-logo.ico binary
 src/etc/pkg/rust-logo.png binary
 *.woff binary
+src/vendor/* binary

.gitignore

@@ -98,3 +98,4 @@ tmp.*.rs
 version.md
 version.ml
 version.texi
+.cargo

.travis.yml

@@ -15,7 +15,7 @@ before_install:
 script:
   - docker run -v `pwd`:/build rust
     sh -c "
-      ./configure --enable-rustbuild --llvm-root=/usr/lib/llvm-3.7 --enable-quiet-tests &&
+      ./configure --enable-vendor --enable-rustbuild --llvm-root=/usr/lib/llvm-3.7 --enable-quiet-tests &&
       make tidy &&
       make check -j4
     "

configure

@@ -634,6 +634,7 @@ opt rustbuild 0 "use the rust and cargo based build system"
 opt codegen-tests 1 "run the src/test/codegen tests"
 opt option-checking 1 "complain about unrecognized options in this configure script"
 opt ninja 0 "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)"
+opt vendor 0 "enable usage of vendored Rust crates"
 
 # Optimization and debugging options. These may be overridden by the release channel, etc.
 opt_nosave optimize 1 "build optimized rust code"

mk/dist.mk

@@ -65,7 +65,8 @@ PKG_FILES := \
       stage0.txt                               \
       rust-installer                           \
       tools                                    \
-      test)                                    \
+      test                                     \
+      vendor)                                  \
     $(PKG_GITMODULES)                          \
     $(filter-out config.stamp, \
                  $(MKFILES_FOR_TARBALL))

src/Cargo.lock

@@ -27,10 +27,6 @@ num_cpus = "0.2"
 toml = "0.1"
 getopts = "0.2"
 rustc-serialize = "0.3"
-gcc = "0.3.36"
+gcc = "0.3.38"
 libc = "0.2"
 md5 = "0.1"
-
-[target.'cfg(windows)'.dependencies]
-winapi = "0.2"
-kernel32-sys = "0.2"

src/bootstrap/Cargo.toml

@@ -259,9 +259,11 @@ def build_bootstrap(self):
         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
                       os.pathsep + env["PATH"]
-        self.run([self.cargo(), "build", "--manifest-path",
-                  os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")],
-                 env)
+        args = [self.cargo(), "build", "--manifest-path",
+                os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
+        if self.use_vendored_sources:
+            args.append("--frozen")
+        self.run(args, env)
 
     def run(self, args, env):
         proc = subprocess.Popen(args, env=env)
@@ -400,6 +402,25 @@ def main():
     except:
         pass
 
+    rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
+                              'CFG_ENABLE_VENDOR' in rb.config_mk
+
+    if rb.use_vendored_sources:
+        if not os.path.exists('.cargo'):
+            os.makedirs('.cargo')
+        f = open('.cargo/config','w')
+        f.write("""
+            [source.crates-io]
+            replace-with = 'vendored-sources'
+            registry = 'https://example.com'
+
+            [source.vendored-sources]
+            directory = '{}/src/vendor'
+        """.format(rb.rust_root))
+        f.close()
+    else:
+        if os.path.exists('.cargo'):
+            shutil.rmtree('.cargo')
     data = stage0_data(rb.rust_root)
     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
     rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)

src/bootstrap/bootstrap.py

@@ -44,6 +44,7 @@ pub struct Config {
     pub submodules: bool,
     pub compiler_docs: bool,
     pub docs: bool,
+    pub vendor: bool,
     pub target_config: HashMap<String, Target>,
 
     // llvm codegen options
@@ -126,6 +127,7 @@ struct Build {
     docs: Option<bool>,
     submodules: Option<bool>,
     gdb: Option<String>,
+    vendor: Option<bool>,
 }
 
 /// TOML representation of how the LLVM build is configured.
@@ -234,6 +236,7 @@ impl Config {
         set(&mut config.compiler_docs, build.compiler_docs);
         set(&mut config.docs, build.docs);
         set(&mut config.submodules, build.submodules);
+        set(&mut config.vendor, build.vendor);
 
         if let Some(ref llvm) = toml.llvm {
             set(&mut config.ccache, llvm.ccache);
@@ -347,6 +350,7 @@ impl Config {
                 ("LOCAL_REBUILD", self.local_rebuild),
                 ("NINJA", self.ninja),
                 ("CODEGEN_TESTS", self.codegen_tests),
+                ("VENDOR", self.vendor),
             }
 
             match key {

src/bootstrap/config.rs

@@ -82,6 +82,9 @@
 # The path to (or name of) the GDB executable to use
 #gdb = "gdb"
 
+# Indicate whether the vendored sources are used for Rust dependencies or not
+#vendor = false
+
 # =============================================================================
 # Options for compiling Rust code itself
 # =============================================================================