rustbuild: Compile rustc twice, not thrice

.travis.yml

@@ -17,6 +17,7 @@ matrix:
     - env: IMAGE=i686-gnu-nopt
     - env: IMAGE=x86_64-freebsd
     - env: IMAGE=x86_64-gnu
+    - env: IMAGE=x86_64-gnu-full-bootstrap
     - env: IMAGE=x86_64-gnu-cargotest
     - env: IMAGE=x86_64-gnu-debug
     - env: IMAGE=x86_64-gnu-nopt

configure

@@ -693,6 +693,7 @@ opt_nosave manage-submodules 1 "let the build manage the git submodules"
 opt_nosave clang 0 "prefer clang to gcc for building the runtime"
 opt_nosave jemalloc 1 "build liballoc with jemalloc"
 opt elf-tls 1 "elf thread local storage on platforms where supported"
+opt full-bootstrap 0 "build three compilers instead of two"
 
 valopt_nosave prefix "/usr/local" "set installation prefix"
 valopt_nosave local-rust-root "/usr/local" "set prefix for local rust binary"

src/bootstrap/check.rs

@@ -341,12 +341,22 @@ pub fn krate(build: &Build,
     println!("{} {} stage{} ({} -> {})", test_kind, name, compiler.stage,
              compiler.host, target);
 
+    // If we're not doing a full bootstrap but we're testing a stage2 version of
+    // libstd, then what we're actually testing is the libstd produced in
+    // stage1. Reflect that here by updating the compiler that we're working
+    // with automatically.
+    let compiler = if build.force_use_stage1(compiler, target) {
+        Compiler::new(1, compiler.host)
+    } else {
+        compiler.clone()
+    };
+
     // Build up the base `cargo test` command.
     //
     // Pass in some standard flags then iterate over the graph we've discovered
     // in `cargo metadata` with the maps above and figure out what `-p`
     // arguments need to get passed.
-    let mut cargo = build.cargo(compiler, mode, target, test_kind.subcommand());
+    let mut cargo = build.cargo(&compiler, mode, target, test_kind.subcommand());
     cargo.arg("--manifest-path")
          .arg(build.src.join(path).join("Cargo.toml"))
          .arg("--features").arg(features);
@@ -380,7 +390,7 @@ pub fn krate(build: &Build,
     // Note that to run the compiler we need to run with the *host* libraries,
     // but our wrapper scripts arrange for that to be the case anyway.
     let mut dylib_path = dylib_path();
-    dylib_path.insert(0, build.sysroot_libdir(compiler, target));
+    dylib_path.insert(0, build.sysroot_libdir(&compiler, target));
     cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
 
     if target.contains("android") {
@@ -399,10 +409,10 @@ pub fn krate(build: &Build,
 
     if target.contains("android") {
         build.run(&mut cargo);
-        krate_android(build, compiler, target, mode);
+        krate_android(build, &compiler, target, mode);
     } else if target.contains("emscripten") {
         build.run(&mut cargo);
-        krate_emscripten(build, compiler, target, mode);
+        krate_emscripten(build, &compiler, target, mode);
     } else {
         cargo.args(&build.flags.cmd.test_args());
         build.run(&mut cargo);

src/bootstrap/compile.rs

@@ -33,14 +33,14 @@ use {Build, Compiler, Mode};
 /// This will build the standard library for a particular stage of the build
 /// using the `compiler` targeting the `target` architecture. The artifacts
 /// created will also be linked into the sysroot directory.
-pub fn std<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
-    println!("Building stage{} std artifacts ({} -> {})", compiler.stage,
-             compiler.host, target);
-
+pub fn std(build: &Build, target: &str, compiler: &Compiler) {
     let libdir = build.sysroot_libdir(compiler, target);
     let _ = fs::remove_dir_all(&libdir);
     t!(fs::create_dir_all(&libdir));
 
+    println!("Building stage{} std artifacts ({} -> {})", compiler.stage,
+             compiler.host, target);
+
     // Some platforms have startup objects that may be required to produce the
     // libstd dynamic library, for example.
     build_startup_objects(build, target, &libdir);
@@ -65,29 +65,30 @@ pub fn std<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
 
     build.run(&mut cargo);
     update_mtime(&libstd_stamp(build, &compiler, target));
-    std_link(build, target, compiler.stage, compiler.host);
 }
 
 /// Link all libstd rlibs/dylibs into the sysroot location.
 ///
-/// Links those artifacts generated in the given `stage` for `target` produced
-/// by `compiler` into `host`'s sysroot.
+/// Links those artifacts generated by `compiler` to a the `stage` compiler's
+/// sysroot for the specified `host` and `target`.
+///
+/// Note that this assumes that `compiler` has already generated the libstd
+/// libraries for `target`, and this method will find them in the relevant
+/// output directory.
 pub fn std_link(build: &Build,
-                target: &str,
-                stage: u32,
-                host: &str) {
-    let compiler = Compiler::new(stage, &build.config.build);
-    let target_compiler = Compiler::new(compiler.stage, host);
+                compiler: &Compiler,
+                target_compiler: &Compiler,
+                target: &str) {
+    println!("Copying stage{} std from stage{} ({} -> {} / {})",
+             target_compiler.stage,
+             compiler.stage,
+             compiler.host,
+             target_compiler.host,
+             target);
     let libdir = build.sysroot_libdir(&target_compiler, target);
     let out_dir = build.cargo_out(&compiler, Mode::Libstd, target);
 
-    // If we're linking one compiler host's output into another, then we weren't
-    // called from the `std` method above. In that case we clean out what's
-    // already there.
-    if host != compiler.host {
-        let _ = fs::remove_dir_all(&libdir);
-        t!(fs::create_dir_all(&libdir));
-    }
+    t!(fs::create_dir_all(&libdir));
     add_to_sysroot(&out_dir, &libdir);
 
     if target.contains("musl") && !target.contains("mips") {
@@ -137,7 +138,7 @@ fn build_startup_objects(build: &Build, target: &str, into: &Path) {
 /// This will build libtest and supporting libraries for a particular stage of
 /// the build using the `compiler` targeting the `target` architecture. The
 /// artifacts created will also be linked into the sysroot directory.
-pub fn test<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
+pub fn test(build: &Build, target: &str, compiler: &Compiler) {
     println!("Building stage{} test artifacts ({} -> {})", compiler.stage,
              compiler.host, target);
     let out_dir = build.cargo_out(compiler, Mode::Libtest, target);
@@ -147,19 +148,13 @@ pub fn test<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
          .arg(build.src.join("src/rustc/test_shim/Cargo.toml"));
     build.run(&mut cargo);
     update_mtime(&libtest_stamp(build, compiler, target));
-    test_link(build, target, compiler.stage, compiler.host);
 }
 
-/// Link all libtest rlibs/dylibs into the sysroot location.
-///
-/// Links those artifacts generated in the given `stage` for `target` produced
-/// by `compiler` into `host`'s sysroot.
+/// Same as `std_link`, only for libtest
 pub fn test_link(build: &Build,
-                 target: &str,
-                 stage: u32,
-                 host: &str) {
-    let compiler = Compiler::new(stage, &build.config.build);
-    let target_compiler = Compiler::new(compiler.stage, host);
+                 compiler: &Compiler,
+                 target_compiler: &Compiler,
+                 target: &str) {
     let libdir = build.sysroot_libdir(&target_compiler, target);
     let out_dir = build.cargo_out(&compiler, Mode::Libtest, target);
     add_to_sysroot(&out_dir, &libdir);
@@ -170,7 +165,7 @@ pub fn test_link(build: &Build,
 /// This will build the compiler for a particular stage of the build using
 /// the `compiler` targeting the `target` architecture. The artifacts
 /// created will also be linked into the sysroot directory.
-pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
+pub fn rustc(build: &Build, target: &str, compiler: &Compiler) {
     println!("Building stage{} compiler artifacts ({} -> {})",
              compiler.stage, compiler.host, target);
 
@@ -222,20 +217,13 @@ pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
         cargo.env("CFG_DEFAULT_AR", s);
     }
     build.run(&mut cargo);
-
-    rustc_link(build, target, compiler.stage, compiler.host);
 }
 
-/// Link all librustc rlibs/dylibs into the sysroot location.
-///
-/// Links those artifacts generated in the given `stage` for `target` produced
-/// by `compiler` into `host`'s sysroot.
+/// Same as `std_link`, only for librustc
 pub fn rustc_link(build: &Build,
-                  target: &str,
-                  stage: u32,
-                  host: &str) {
-    let compiler = Compiler::new(stage, &build.config.build);
-    let target_compiler = Compiler::new(compiler.stage, host);
+                  compiler: &Compiler,
+                  target_compiler: &Compiler,
+                  target: &str) {
     let libdir = build.sysroot_libdir(&target_compiler, target);
     let out_dir = build.cargo_out(&compiler, Mode::Librustc, target);
     add_to_sysroot(&out_dir, &libdir);
@@ -259,6 +247,17 @@ fn compiler_file(compiler: &Path, file: &str) -> PathBuf {
     PathBuf::from(out.trim())
 }
 
+pub fn create_sysroot(build: &Build, compiler: &Compiler) {
+    // nothing to do in stage0
+    if compiler.stage == 0 {
+        return
+    }
+
+    let sysroot = build.sysroot(compiler);
+    let _ = fs::remove_dir_all(&sysroot);
+    t!(fs::create_dir_all(&sysroot));
+}
+
 /// Prepare a new compiler from the artifacts in `stage`
 ///
 /// This will assemble a compiler in `build/$host/stage$stage`. The compiler
@@ -269,18 +268,17 @@ pub fn assemble_rustc(build: &Build, stage: u32, host: &str) {
     if stage == 0 {
         return
     }
+
+    println!("Copying stage{} compiler ({})", stage, host);
+
     // The compiler that we're assembling
     let target_compiler = Compiler::new(stage, host);
 
     // The compiler that compiled the compiler we're assembling
     let build_compiler = Compiler::new(stage - 1, &build.config.build);
 
-    // Clear out old files
-    let sysroot = build.sysroot(&target_compiler);
-    let _ = fs::remove_dir_all(&sysroot);
-    t!(fs::create_dir_all(&sysroot));
-
     // Link in all dylibs to the libdir
+    let sysroot = build.sysroot(&target_compiler);
     let sysroot_libdir = sysroot.join(libdir(host));
     t!(fs::create_dir_all(&sysroot_libdir));
     let src_libdir = build.sysroot_libdir(&build_compiler, host);

src/bootstrap/config.rs

@@ -46,6 +46,7 @@ pub struct Config {
     pub docs: bool,
     pub vendor: bool,
     pub target_config: HashMap<String, Target>,
+    pub full_bootstrap: bool,
 
     // llvm codegen options
     pub llvm_assertions: bool,
@@ -134,6 +135,7 @@ struct Build {
     vendor: Option<bool>,
     nodejs: Option<String>,
     python: Option<String>,
+    full_bootstrap: Option<bool>,
 }
 
 /// TOML representation of various global install decisions.
@@ -264,6 +266,7 @@ impl Config {
         set(&mut config.docs, build.docs);
         set(&mut config.submodules, build.submodules);
         set(&mut config.vendor, build.vendor);
+        set(&mut config.full_bootstrap, build.full_bootstrap);
 
         if let Some(ref install) = toml.install {
             config.prefix = install.prefix.clone();
@@ -393,6 +396,7 @@ impl Config {
                 ("NINJA", self.ninja),
                 ("CODEGEN_TESTS", self.codegen_tests),
                 ("VENDOR", self.vendor),
+                ("FULL_BOOTSTRAP", self.full_bootstrap),
             }
 
             match key {

src/bootstrap/config.toml.example

@@ -100,6 +100,13 @@
 # Indicate whether the vendored sources are used for Rust dependencies or not
 #vendor = false
 
+# Typically the build system will build the rust compiler twice. The second
+# compiler, however, will simply use its own libraries to link against. If you
+# would rather to perform a full bootstrap, compiling the compiler three times,
+# then you can set this option to true. You shouldn't ever need to set this
+# option to true.
+#full-bootstrap = false
+
 # =============================================================================
 # General install configuration options
 # =============================================================================

src/bootstrap/doc.rs

@@ -137,6 +137,11 @@ pub fn std(build: &Build, stage: u32, target: &str) {
     let out = build.doc_out(target);
     t!(fs::create_dir_all(&out));
     let compiler = Compiler::new(stage, &build.config.build);
+    let compiler = if build.force_use_stage1(&compiler, target) {
+        Compiler::new(1, compiler.host)
+    } else {
+        compiler
+    };
     let out_dir = build.stage_out(&compiler, Mode::Libstd)
                        .join(target).join("doc");
     let rustdoc = build.rustdoc(&compiler);
@@ -160,6 +165,11 @@ pub fn test(build: &Build, stage: u32, target: &str) {
     let out = build.doc_out(target);
     t!(fs::create_dir_all(&out));
     let compiler = Compiler::new(stage, &build.config.build);
+    let compiler = if build.force_use_stage1(&compiler, target) {
+        Compiler::new(1, compiler.host)
+    } else {
+        compiler
+    };
     let out_dir = build.stage_out(&compiler, Mode::Libtest)
                        .join(target).join("doc");
     let rustdoc = build.rustdoc(&compiler);
@@ -182,6 +192,11 @@ pub fn rustc(build: &Build, stage: u32, target: &str) {
     let out = build.doc_out(target);
     t!(fs::create_dir_all(&out));
     let compiler = Compiler::new(stage, &build.config.build);
+    let compiler = if build.force_use_stage1(&compiler, target) {
+        Compiler::new(1, compiler.host)
+    } else {
+        compiler
+    };
     let out_dir = build.stage_out(&compiler, Mode::Librustc)
                        .join(target).join("doc");
     let rustdoc = build.rustdoc(&compiler);

src/bootstrap/lib.rs

@@ -572,9 +572,7 @@ impl Build {
         let mut cmd = Command::new(self.tool(&compiler, tool));
         let host = compiler.host;
         let mut paths = vec![
-            self.cargo_out(compiler, Mode::Libstd, host).join("deps"),
-            self.cargo_out(compiler, Mode::Libtest, host).join("deps"),
-            self.cargo_out(compiler, Mode::Librustc, host).join("deps"),
+            self.sysroot_libdir(compiler, compiler.host),
             self.cargo_out(compiler, Mode::Tool, host).join("deps"),
         ];
 
@@ -880,6 +878,30 @@ impl Build {
     fn python(&self) -> &Path {
         self.config.python.as_ref().unwrap()
     }
+
+    /// Tests whether the `compiler` compiling for `target` should be forced to
+    /// use a stage1 compiler instead.
+    ///
+    /// Currently, by default, the build system does not perform a "full
+    /// bootstrap" by default where we compile the compiler three times.
+    /// Instead, we compile the compiler two times. The final stage (stage2)
+    /// just copies the libraries from the previous stage, which is what this
+    /// method detects.
+    ///
+    /// Here we return `true` if:
+    ///
+    /// * The build isn't performing a full bootstrap
+    /// * The `compiler` is in the final stage, 2
+    /// * We're not cross-compiling, so the artifacts are already available in
+    ///   stage1
+    ///
+    /// When all of these conditions are met the build will lift artifacts from
+    /// the previous stage forward.
+    fn force_use_stage1(&self, compiler: &Compiler, target: &str) -> bool {
+        !self.config.full_bootstrap &&
+            compiler.stage >= 2 &&
+            self.config.host.iter().any(|h| h == target)
+    }
 }
 
 impl<'a> Compiler<'a> {