Auto merge of #40329 - petrochenkov:llreuse, r=alexcrichton

rustbuild: Add option for enabling partial LLVM rebuilds

@alexcrichton , you probably didn't notice my [late comment](https://github.com/rust-lang/rust/pull/40236#issuecomment-284160749) on https://github.com/rust-lang/rust/pull/40236, but here's an implementation of that suggestion, it supersedes c652a4fb56.

r? @alexcrichton

appveyor.yml

@@ -144,10 +144,10 @@ on_failure:
   - cat %CD%/sccache.log
 
 cache:
-  - "build/i686-pc-windows-msvc/llvm -> src/rustllvm/llvm-auto-clean-trigger"
-  - "build/x86_64-pc-windows-msvc/llvm -> src/rustllvm/llvm-auto-clean-trigger"
-  - "i686-pc-windows-msvc/llvm -> src/rustllvm/llvm-auto-clean-trigger"
-  - "x86_64-pc-windows-msvc/llvm -> src/rustllvm/llvm-auto-clean-trigger"
+  - "build/i686-pc-windows-msvc/llvm -> src/rustllvm/llvm-rebuild-trigger"
+  - "build/x86_64-pc-windows-msvc/llvm -> src/rustllvm/llvm-rebuild-trigger"
+  - "i686-pc-windows-msvc/llvm -> src/rustllvm/llvm-rebuild-trigger"
+  - "x86_64-pc-windows-msvc/llvm -> src/rustllvm/llvm-rebuild-trigger"
 
 branches:
   only:

configure

@@ -437,6 +437,7 @@ opt local-rust 0 "use an installed rustc rather than downloading a snapshot"
 opt local-rebuild 0 "assume local-rust matches the current version, for rebuilds; implies local-rust, and is implied if local-rust already matches the current version"
 opt llvm-static-stdcpp 0 "statically link to libstdc++ for LLVM"
 opt llvm-link-shared 0 "prefer shared linking to LLVM (llvm-config --link-shared)"
+opt llvm-clean-rebuild 0 "delete LLVM build directory on rebuild"
 opt rpath 1 "build rpaths into rustc itself"
 opt stage0-landing-pads 1 "enable landing pads during bootstrap with stage0"
 # This is used by the automation to produce single-target nightlies

src/bootstrap/config.rs

@@ -60,6 +60,7 @@ pub struct Config {
     pub llvm_link_shared: bool,
     pub llvm_targets: Option<String>,
     pub llvm_link_jobs: Option<u32>,
+    pub llvm_clean_rebuild: bool,
 
     // rust codegen options
     pub rust_optimize: bool,
@@ -181,6 +182,7 @@ struct Llvm {
     static_libstdcpp: Option<bool>,
     targets: Option<String>,
     link_jobs: Option<u32>,
+    clean_rebuild: Option<bool>,
 }
 
 #[derive(RustcDecodable, Default, Clone)]
@@ -334,6 +336,7 @@ pub fn parse(build: &str, file: Option<PathBuf>) -> Config {
             set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo);
             set(&mut config.llvm_version_check, llvm.version_check);
             set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
+            set(&mut config.llvm_clean_rebuild, llvm.clean_rebuild);
             config.llvm_targets = llvm.targets.clone();
             config.llvm_link_jobs = llvm.link_jobs;
         }
@@ -439,6 +442,7 @@ macro_rules! check {
                 ("LLVM_VERSION_CHECK", self.llvm_version_check),
                 ("LLVM_STATIC_STDCPP", self.llvm_static_stdcpp),
                 ("LLVM_LINK_SHARED", self.llvm_link_shared),
+                ("LLVM_CLEAN_REBUILD", self.llvm_clean_rebuild),
                 ("OPTIMIZE", self.rust_optimize),
                 ("DEBUG_ASSERTIONS", self.rust_debug_assertions),
                 ("DEBUGINFO", self.rust_debuginfo),

src/bootstrap/config.toml.example

@@ -61,6 +61,11 @@
 # controlled by rustbuild's -j parameter.
 #link-jobs = 0
 
+# Delete LLVM build directory on LLVM rebuild.
+# This option defaults to `false` for local development, but CI may want to
+# always perform clean full builds (possibly accelerated by (s)ccache).
+#clean-rebuild = false
+
 # =============================================================================
 # General build configuration options
 # =============================================================================

src/bootstrap/native.rs

@@ -41,9 +41,9 @@ pub fn llvm(build: &Build, target: &str) {
         }
     }
 
-    let clean_trigger = build.src.join("src/rustllvm/llvm-auto-clean-trigger");
-    let mut clean_trigger_contents = String::new();
-    t!(t!(File::open(&clean_trigger)).read_to_string(&mut clean_trigger_contents));
+    let rebuild_trigger = build.src.join("src/rustllvm/llvm-rebuild-trigger");
+    let mut rebuild_trigger_contents = String::new();
+    t!(t!(File::open(&rebuild_trigger)).read_to_string(&mut rebuild_trigger_contents));
 
     let out_dir = build.llvm_out(target);
     let done_stamp = out_dir.join("llvm-finished-building");
@@ -51,18 +51,15 @@ pub fn llvm(build: &Build, target: &str) {
         let mut done_contents = String::new();
         t!(t!(File::open(&done_stamp)).read_to_string(&mut done_contents));
 
-        // LLVM was already built previously.
-        // We don't track changes in LLVM sources, so we need to choose between reusing
-        // what was built previously, or cleaning the directory and doing a fresh build.
-        // The choice depends on contents of the clean-trigger file.
-        // If the contents are the same as during the previous build, then no action is required.
-        // If the contents differ from the previous build, then cleaning is triggered.
-        if done_contents == clean_trigger_contents {
+        // If LLVM was already built previously and contents of the rebuild-trigger file
+        // didn't change from the previous build, then no action is required.
+        if done_contents == rebuild_trigger_contents {
             return
-        } else {
-            t!(fs::remove_dir_all(&out_dir));
         }
     }
+    if build.config.llvm_clean_rebuild {
+        drop(fs::remove_dir_all(&out_dir));
+    }
 
     println!("Building LLVM for {}", target);
     let _time = util::timeit();
@@ -154,7 +151,7 @@ pub fn llvm(build: &Build, target: &str) {
     //        tools and libs on all platforms.
     cfg.build();
 
-    t!(t!(File::create(&done_stamp)).write_all(clean_trigger_contents.as_bytes()));
+    t!(t!(File::create(&done_stamp)).write_all(rebuild_trigger_contents.as_bytes()));
 }
 
 fn check_llvm_version(build: &Build, llvm_config: &Path) {

src/ci/run.sh

@@ -28,6 +28,7 @@ RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-quiet-tests"
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-manage-submodules"
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-locked-deps"
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-cargo-openssl-static"
+RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-clean-rebuild"
 
 if [ "$DIST_SRC" = "" ]; then
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-dist-src"

src/rustllvm/llvm-rebuild-trigger

@@ -1,4 +1,4 @@
-# If this file is modified, then llvm will be forcibly cleaned and then rebuilt.
+# If this file is modified, then llvm will be (optionally) cleaned and then rebuilt.
 # The actual contents of this file do not matter, but to trigger a change on the
 # build bots then the contents should be changed so git updates the mtime.
 2017-03-04