From 5e577b8aee60b7ee2394a50d8eec8a3b26541d11 Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Mon, 30 Apr 2018 10:15:48 +0200 Subject: [PATCH 1/8] ship LLVM tools with the toolchain --- config.toml.example | 4 ++++ src/bootstrap/compile.rs | 22 +++++++++++++++++++++- src/bootstrap/config.rs | 3 +++ src/bootstrap/configure.py | 1 + src/bootstrap/dist.rs | 20 +++++++++++++++++++- src/bootstrap/lib.rs | 2 ++ src/bootstrap/native.rs | 7 +++++-- 7 files changed, 55 insertions(+), 4 deletions(-) diff --git a/config.toml.example b/config.toml.example index 33ad9147ce059..cbf209932099e 100644 --- a/config.toml.example +++ b/config.toml.example @@ -347,6 +347,10 @@ # rustc to execute. #lld = false +# Indicates whether some LLVM tools, like llvm-objdump, will be made available in the +# sysroot. +#llvm-tools = false + # Whether to deny warnings in crates #deny-warnings = true diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 231ed9d40d2de..ea234c0016c34 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -31,7 +31,7 @@ use filetime::FileTime; use serde_json; use util::{exe, libdir, is_dylib, CiEnv}; -use {Compiler, Mode}; +use {Compiler, Mode, LLVM_TOOLS}; use native; use tool; @@ -775,6 +775,23 @@ fn copy_codegen_backends_to_sysroot(builder: &Builder, } } +fn copy_llvm_tools_to_sysroot(builder: &Builder, + target_compiler: Compiler) { + let target = target_compiler.host; + + let dst = builder.sysroot_libdir(target_compiler, target) + .parent() + .unwrap() + .join("bin"); + t!(fs::create_dir_all(&dst)); + + let src = builder.llvm_out(target).join("bin"); + for tool in LLVM_TOOLS { + let exe = exe(tool, &target); + builder.copy(&src.join(&exe), &dst.join(&exe)); + } +} + fn copy_lld_to_sysroot(builder: &Builder, target_compiler: Compiler, lld_install_root: &Path) { @@ -966,6 +983,9 @@ impl Step for Assemble { copy_codegen_backends_to_sysroot(builder, build_compiler, target_compiler); + if builder.config.ship_llvm_tools { + copy_llvm_tools_to_sysroot(builder, target_compiler); + } if let Some(lld_install) = lld_install { copy_lld_to_sysroot(builder, target_compiler, &lld_install); } diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 9840682d1379b..b676ba22172b5 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -88,6 +88,7 @@ pub struct Config { pub llvm_link_jobs: Option, pub lld_enabled: bool, + pub ship_llvm_tools: bool, // rust codegen options pub rust_optimize: bool, @@ -308,6 +309,7 @@ struct Rust { codegen_backends_dir: Option, wasm_syscall: Option, lld: Option, + llvm_tools: Option, deny_warnings: Option, backtrace_on_ice: Option, } @@ -531,6 +533,7 @@ impl Config { set(&mut config.test_miri, rust.test_miri); set(&mut config.wasm_syscall, rust.wasm_syscall); set(&mut config.lld_enabled, rust.lld); + set(&mut config.ship_llvm_tools, rust.llvm_tools); config.rustc_parallel_queries = rust.experimental_parallel_queries.unwrap_or(false); config.rustc_default_linker = rust.default_linker.clone(); config.musl_root = rust.musl_root.clone().map(PathBuf::from); diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index 3574b7d210a2d..36d936bab3baa 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -335,6 +335,7 @@ def set(key, value): elif option.name == 'full-tools': set('rust.codegen-backends', ['llvm', 'emscripten']) set('rust.lld', True) + set('rust.llvm-tools', True) set('build.extended', True) elif option.name == 'option-checking': # this was handled above diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 82ba03ec7773c..9092deeac4618 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -26,7 +26,7 @@ use std::process::{Command, Stdio}; use build_helper::output; -use {Compiler, Mode}; +use {Compiler, Mode, LLVM_TOOLS}; use channel; use util::{libdir, is_dylib, exe}; use builder::{Builder, RunConfig, ShouldRun, Step}; @@ -503,6 +503,24 @@ impl Step for Rustc { builder.copy(&src, &dst); } + if builder.config.ship_llvm_tools { + let src = builder.sysroot_libdir(compiler, host) + .parent() + .unwrap() + .join("bin"); + + let dst = image.join("lib/rustlib") + .join(&*host) + .join("bin"); + + t!(fs::create_dir_all(&dst.parent().unwrap())); + + for tool in LLVM_TOOLS { + let exe = exe(tool, &compiler.host); + builder.copy(&src.join(&exe), &dst.join(&exe)); + } + } + // Man pages t!(fs::create_dir_all(image.join("share/man/man1"))); let man_src = builder.src.join("src/doc/man"); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index f64161fb0272c..4d1bc6ad6bae3 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -199,6 +199,8 @@ use flags::Subcommand; use cache::{Interned, INTERNER}; use toolstate::ToolState; +const LLVM_TOOLS: &[&str] = &["llvm-nm", "llvm-objcopy", "llvm-objdump", "llvm-size"]; + /// A structure representing a Rust compiler. /// /// Each compiler has a `stage` that it is associated with and a `host` that diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 93292c658bad2..2f6af1e064062 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -167,8 +167,11 @@ impl Step for Llvm { // which saves both memory during parallel links and overall disk space // for the tools. We don't distribute any of those tools, so this is // just a local concern. However, it doesn't work well everywhere. - if target.contains("linux-gnu") || target.contains("apple-darwin") { - cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); + // + // If we are shipping llvm tools then we statically link them LLVM + if (target.contains("linux-gnu") || target.contains("apple-darwin")) && + !builder.config.ship_llvm_tools { + cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); } if target.contains("msvc") { From b5cdf9dbf9c0a75f8dab5d6f678fa68cc9a25ee1 Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Mon, 30 Apr 2018 13:29:45 +0200 Subject: [PATCH 2/8] also ship llvm-profdata --- src/bootstrap/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 4d1bc6ad6bae3..c271af3f035bd 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -199,7 +199,9 @@ use flags::Subcommand; use cache::{Interned, INTERNER}; use toolstate::ToolState; -const LLVM_TOOLS: &[&str] = &["llvm-nm", "llvm-objcopy", "llvm-objdump", "llvm-size"]; +const LLVM_TOOLS: &[&str] = &[ + "llvm-nm", "llvm-objcopy", "llvm-objdump", "llvm-profdata", "llvm-size", +]; /// A structure representing a Rust compiler. /// From a1ef529703172a1eb432782b3494512b74af6b07 Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Tue, 1 May 2018 06:34:12 +0200 Subject: [PATCH 3/8] document what each tools does --- src/bootstrap/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index c271af3f035bd..dec514a0da6e6 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -200,7 +200,11 @@ use cache::{Interned, INTERNER}; use toolstate::ToolState; const LLVM_TOOLS: &[&str] = &[ - "llvm-nm", "llvm-objcopy", "llvm-objdump", "llvm-profdata", "llvm-size", + "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility + "llvm-objcopy", // used to transform ELFs into binary format which flashing tools consume + "llvm-objdump", // used to disassemble programs + "llvm-profdata", // used to inspect and merge files generated by profiles + "llvm-size", // prints the size of the linker sections of a program ]; /// A structure representing a Rust compiler. From 8e673073f349f9b7cadaf809f8b88a2bfd296769 Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Tue, 1 May 2018 06:34:24 +0200 Subject: [PATCH 4/8] statically link the tools to libstdc++ --- src/bootstrap/native.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 2f6af1e064062..74083360aca78 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -174,6 +174,11 @@ impl Step for Llvm { cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); } + // For distribution we want the LLVM tools to be *statically* linked to libstdc++ + if builder.config.ship_llvm_tools { + cfg.define("CMAKE_EXE_LINKER_FLAGS", "-Wl,-Bsymbolic -static-libstdc++"); + } + if target.contains("msvc") { cfg.define("LLVM_USE_CRT_DEBUG", "MT"); cfg.define("LLVM_USE_CRT_RELEASE", "MT"); From 66a7db9a3593b1bc4744239a21520369f0418d3c Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Wed, 30 May 2018 08:01:35 +0200 Subject: [PATCH 5/8] make a llvm-tools rustup component --- src/bootstrap/builder.rs | 1 + src/bootstrap/compile.rs | 22 +-------- src/bootstrap/config.rs | 4 +- src/bootstrap/dist.rs | 98 ++++++++++++++++++++++++++++++++-------- src/bootstrap/lib.rs | 18 ++++++++ src/bootstrap/native.rs | 4 +- 6 files changed, 103 insertions(+), 44 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index c9c9c73c84af2..e0932dbf91a13 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -451,6 +451,7 @@ impl<'a> Builder<'a> { dist::Cargo, dist::Rls, dist::Rustfmt, + dist::LlvmTools, dist::Extended, dist::HashSign ), diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index ea234c0016c34..231ed9d40d2de 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -31,7 +31,7 @@ use filetime::FileTime; use serde_json; use util::{exe, libdir, is_dylib, CiEnv}; -use {Compiler, Mode, LLVM_TOOLS}; +use {Compiler, Mode}; use native; use tool; @@ -775,23 +775,6 @@ fn copy_codegen_backends_to_sysroot(builder: &Builder, } } -fn copy_llvm_tools_to_sysroot(builder: &Builder, - target_compiler: Compiler) { - let target = target_compiler.host; - - let dst = builder.sysroot_libdir(target_compiler, target) - .parent() - .unwrap() - .join("bin"); - t!(fs::create_dir_all(&dst)); - - let src = builder.llvm_out(target).join("bin"); - for tool in LLVM_TOOLS { - let exe = exe(tool, &target); - builder.copy(&src.join(&exe), &dst.join(&exe)); - } -} - fn copy_lld_to_sysroot(builder: &Builder, target_compiler: Compiler, lld_install_root: &Path) { @@ -983,9 +966,6 @@ impl Step for Assemble { copy_codegen_backends_to_sysroot(builder, build_compiler, target_compiler); - if builder.config.ship_llvm_tools { - copy_llvm_tools_to_sysroot(builder, target_compiler); - } if let Some(lld_install) = lld_install { copy_lld_to_sysroot(builder, target_compiler, &lld_install); } diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index b676ba22172b5..c6b1b87247003 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -88,7 +88,7 @@ pub struct Config { pub llvm_link_jobs: Option, pub lld_enabled: bool, - pub ship_llvm_tools: bool, + pub llvm_tools_enabled: bool, // rust codegen options pub rust_optimize: bool, @@ -533,7 +533,7 @@ impl Config { set(&mut config.test_miri, rust.test_miri); set(&mut config.wasm_syscall, rust.wasm_syscall); set(&mut config.lld_enabled, rust.lld); - set(&mut config.ship_llvm_tools, rust.llvm_tools); + set(&mut config.llvm_tools_enabled, rust.llvm_tools); config.rustc_parallel_queries = rust.experimental_parallel_queries.unwrap_or(false); config.rustc_default_linker = rust.default_linker.clone(); config.musl_root = rust.musl_root.clone().map(PathBuf::from); diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 9092deeac4618..73cda4b436d42 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -43,6 +43,8 @@ pub fn pkgname(builder: &Builder, component: &str) -> String { format!("{}-{}", component, builder.rls_package_vers()) } else if component == "rustfmt" { format!("{}-{}", component, builder.rustfmt_package_vers()) + } else if component == "llvm-tools" { + format!("{}-{}", component, builder.llvm_tools_vers()) } else { assert!(component.starts_with("rust")); format!("{}-{}", component, builder.rust_package_vers()) @@ -394,7 +396,7 @@ impl Step for Rustc { let compiler = self.compiler; let host = self.compiler.host; - builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, compiler.host)); + builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host)); let name = pkgname(builder, "rustc"); let image = tmpdir(builder).join(format!("{}-{}-image", name, host)); let _ = fs::remove_dir_all(&image); @@ -503,24 +505,6 @@ impl Step for Rustc { builder.copy(&src, &dst); } - if builder.config.ship_llvm_tools { - let src = builder.sysroot_libdir(compiler, host) - .parent() - .unwrap() - .join("bin"); - - let dst = image.join("lib/rustlib") - .join(&*host) - .join("bin"); - - t!(fs::create_dir_all(&dst.parent().unwrap())); - - for tool in LLVM_TOOLS { - let exe = exe(tool, &compiler.host); - builder.copy(&src.join(&exe), &dst.join(&exe)); - } - } - // Man pages t!(fs::create_dir_all(image.join("share/man/man1"))); let man_src = builder.src.join("src/doc/man"); @@ -1756,6 +1740,7 @@ impl Step for HashSign { cmd.arg(builder.package_vers(&builder.release_num("cargo"))); cmd.arg(builder.package_vers(&builder.release_num("rls"))); cmd.arg(builder.package_vers(&builder.release_num("rustfmt"))); + cmd.arg(builder.llvm_tools_vers()); cmd.arg(addr); builder.create_dir(&distdir(builder)); @@ -1766,3 +1751,78 @@ impl Step for HashSign { assert!(status.success()); } } + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct LlvmTools { + pub stage: u32, + pub compiler: Compiler, + pub target: Interned, +} + +impl Step for LlvmTools { + type Output = Option; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun) -> ShouldRun { + run.path("llvm-tools") + } + + fn make_run(run: RunConfig) { + run.builder.ensure(LlvmTools { + stage: run.builder.top_stage, + compiler: run.builder.compiler(run.builder.top_stage, run.target), + target: run.target, + }); + } + + fn run(self, builder: &Builder) -> Option { + let compiler = self.compiler; + let host = compiler.host; + + let stage = self.stage; + assert!(builder.config.extended); + + builder.info(&format!("Dist LlvmTools stage{} ({})", stage, host)); + let src = builder.src.join("src/llvm"); + let name = pkgname(builder, "llvm-tools"); + + let tmp = tmpdir(builder); + let image = tmp.join("llvm-tools-image"); + drop(fs::remove_dir_all(&image)); + t!(fs::create_dir_all(&image.join("bin"))); + + // Prepare the image directory + for tool in LLVM_TOOLS { + let exe = builder + .llvm_out(host) + .join("bin") + .join(exe(tool, &compiler.host)); + builder.install(&exe, &image.join("bin"), 0o755); + } + + // Prepare the overlay + let overlay = tmp.join("llvm-tools-overlay"); + drop(fs::remove_dir_all(&overlay)); + builder.create_dir(&overlay); + builder.install(&src.join("README.txt"), &overlay, 0o644); + builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644); + + // Generate the installer tarball + let mut cmd = rust_installer(builder); + cmd.arg("generate") + .arg("--product-name=Rust") + .arg("--rel-manifest-dir=rustlib") + .arg("--success-message=llvm-tools-installed.") + .arg("--image-dir").arg(&image) + .arg("--work-dir").arg(&tmpdir(builder)) + .arg("--output-dir").arg(&distdir(builder)) + .arg("--non-installed-overlay").arg(&overlay) + .arg(format!("--package-name={}-{}", name, host)) + .arg("--legacy-manifest-dirs=rustlib,cargo") + .arg("--component-name=llvm-tools"); + + + builder.run(&mut cmd); + Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host))) + } +} diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index dec514a0da6e6..cb135ad6f3523 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -957,6 +957,24 @@ impl Build { self.package_vers(&self.release_num("rustfmt")) } + fn llvm_tools_vers(&self) -> String { + let stdout = build_helper::output( + Command::new(self.llvm_out(self.config.build).join("build/bin/llvm-size")) + .arg("--version"), + ); + + for line in stdout.lines() { + if line.contains("LLVM version") { + if let Some(vers) = line.split_whitespace().nth(2) { + return vers.to_string(); + } + } + } + + panic!("The output of $LLVM_TOOL has changed; \ + please fix `bootstrap::Build.llvm_tools_vers`"); + } + /// Returns the `version` string associated with this compiler for Rust /// itself. /// diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 74083360aca78..c3f84378adfb2 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -170,12 +170,12 @@ impl Step for Llvm { // // If we are shipping llvm tools then we statically link them LLVM if (target.contains("linux-gnu") || target.contains("apple-darwin")) && - !builder.config.ship_llvm_tools { + !builder.config.llvm_tools_enabled { cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); } // For distribution we want the LLVM tools to be *statically* linked to libstdc++ - if builder.config.ship_llvm_tools { + if builder.config.llvm_tools_enabled { cfg.define("CMAKE_EXE_LINKER_FLAGS", "-Wl,-Bsymbolic -static-libstdc++"); } From a2c2ed3c0cba4485d97219c2f4e25f0e4f7654d4 Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Sun, 3 Jun 2018 13:56:58 +0200 Subject: [PATCH 6/8] use rustc version as llvm-tools version --- src/bootstrap/lib.rs | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index cb135ad6f3523..5bb9584a8da1c 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -958,21 +958,24 @@ impl Build { } fn llvm_tools_vers(&self) -> String { - let stdout = build_helper::output( - Command::new(self.llvm_out(self.config.build).join("build/bin/llvm-size")) - .arg("--version"), - ); - - for line in stdout.lines() { - if line.contains("LLVM version") { - if let Some(vers) = line.split_whitespace().nth(2) { - return vers.to_string(); - } - } - } - - panic!("The output of $LLVM_TOOL has changed; \ - please fix `bootstrap::Build.llvm_tools_vers`"); + // XXX should we use LLVM version here? + // let stdout = build_helper::output( + // Command::new(self.llvm_out(self.config.build).join("build/bin/llvm-size")) + // .arg("--version"), + // ); + + // for line in stdout.lines() { + // if line.contains("LLVM version") { + // if let Some(vers) = line.split_whitespace().nth(2) { + // return vers.to_string(); + // } + // } + // } + + // panic!("The output of $LLVM_TOOL has changed; \ + // please fix `bootstrap::Build.llvm_tools_vers`"); + + self.rust_version() } /// Returns the `version` string associated with this compiler for Rust From 7458af99e922036440648a157ade4899f23dd4a3 Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Mon, 4 Jun 2018 00:25:43 +0200 Subject: [PATCH 7/8] fix tidy --- src/bootstrap/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 5bb9584a8da1c..996767d97cf4a 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -958,7 +958,7 @@ impl Build { } fn llvm_tools_vers(&self) -> String { - // XXX should we use LLVM version here? + // japaric: should we use LLVM version here? // let stdout = build_helper::output( // Command::new(self.llvm_out(self.config.build).join("build/bin/llvm-size")) // .arg("--version"), From 9a96876d2d1f8e2a4ca066f132c4b6ddec1b94ff Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Wed, 20 Jun 2018 17:48:04 -0500 Subject: [PATCH 8/8] no -Bsymbolic for mac; no static-libstdc++ for windows --- src/bootstrap/native.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index c3f84378adfb2..7dcdbe9c931cf 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -176,7 +176,13 @@ impl Step for Llvm { // For distribution we want the LLVM tools to be *statically* linked to libstdc++ if builder.config.llvm_tools_enabled { - cfg.define("CMAKE_EXE_LINKER_FLAGS", "-Wl,-Bsymbolic -static-libstdc++"); + if !target.contains("windows") { + if target.contains("apple") { + cfg.define("CMAKE_EXE_LINKER_FLAGS", "-static-libstdc++"); + } else { + cfg.define("CMAKE_EXE_LINKER_FLAGS", "-Wl,-Bsymbolic -static-libstdc++"); + } + } } if target.contains("msvc") {