From ddf82636c6b2d42a1fe9d25b51ec9b006043f529 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 26 Nov 2020 16:52:20 +0100 Subject: [PATCH 01/23] bootstrap: convert build-manifest to use the new Tarball struct --- src/bootstrap/dist.rs | 109 +++++++--------------------------- src/bootstrap/lib.rs | 1 + src/bootstrap/tarball.rs | 124 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 88 deletions(-) create mode 100644 src/bootstrap/tarball.rs diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index fcbde9c438cb5..0ec896ed2115e 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -19,6 +19,7 @@ use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::cache::{Interned, INTERNER}; use crate::compile; use crate::config::TargetSelection; +use crate::tarball::{OverlayKind, Tarball}; use crate::tool::{self, Tool}; use crate::util::{exe, is_dylib, timeit}; use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS}; @@ -2517,68 +2518,36 @@ impl Step for RustDev { builder.info(&format!("Dist RustDev ({})", target)); let _time = timeit(builder); - let src = builder.src.join("src/llvm-project/llvm"); - let name = pkgname(builder, "rust-dev"); - - let tmp = tmpdir(builder); - let image = tmp.join("rust-dev-image"); - drop(fs::remove_dir_all(&image)); - // Prepare the image directory - let dst_bindir = image.join("bin"); - t!(fs::create_dir_all(&dst_bindir)); + let mut tarball = Tarball::new(builder, "rust-dev", &target.triple); + tarball.set_overlay(OverlayKind::LLVM); let src_bindir = builder.llvm_out(target).join("bin"); - let install_bin = - |name| builder.install(&src_bindir.join(exe(name, target)), &dst_bindir, 0o755); - install_bin("llvm-config"); - install_bin("llvm-ar"); - install_bin("llvm-objdump"); - install_bin("llvm-profdata"); - install_bin("llvm-bcanalyzer"); - install_bin("llvm-cov"); - install_bin("llvm-dwp"); - builder.install(&builder.llvm_filecheck(target), &dst_bindir, 0o755); + for bin in &[ + "llvm-config", + "llvm-ar", + "llvm-objdump", + "llvm-profdata", + "llvm-bcanalyzer", + "llvm-cov", + "llvm-dwp", + ] { + tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755); + } + tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755); // Copy the include directory as well; needed mostly to build // librustc_llvm properly (e.g., llvm-config.h is in here). But also // just broadly useful to be able to link against the bundled LLVM. - builder.cp_r(&builder.llvm_out(target).join("include"), &image.join("include")); + tarball.add_dir(&builder.llvm_out(target).join("include"), "."); // Copy libLLVM.so to the target lib dir as well, so the RPATH like // `$ORIGIN/../lib` can find it. It may also be used as a dependency // of `rustc-dev` to support the inherited `-lLLVM` when using the // compiler libraries. - maybe_install_llvm(builder, target, &image.join("lib")); - - // Prepare the overlay - let overlay = tmp.join("rust-dev-overlay"); - drop(fs::remove_dir_all(&overlay)); - builder.create_dir(&overlay); - builder.install(&src.join("README.txt"), &overlay, 0o644); - builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644); - builder.create(&overlay.join("version"), &builder.rust_version()); + maybe_install_llvm(builder, target, &tarball.image_dir().join("lib")); - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=rust-dev-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=rust-dev"); - - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + Some(tarball.generate()) } } @@ -2607,45 +2576,9 @@ impl Step for BuildManifest { fn run(self, builder: &Builder<'_>) -> PathBuf { let build_manifest = builder.tool_exe(Tool::BuildManifest); - let name = pkgname(builder, "build-manifest"); - let tmp = tmpdir(builder); - - // Prepare the image. - let image = tmp.join("build-manifest-image"); - let image_bin = image.join("bin"); - let _ = fs::remove_dir_all(&image); - t!(fs::create_dir_all(&image_bin)); - builder.install(&build_manifest, &image_bin, 0o755); - - // Prepare the overlay. - let overlay = tmp.join("build-manifest-overlay"); - let _ = fs::remove_dir_all(&overlay); - builder.create_dir(&overlay); - builder.create(&overlay.join("version"), &builder.rust_version()); - for file in &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"] { - builder.install(&builder.src.join(file), &overlay, 0o644); - } - - // Create the final tarball. - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=build-manifest installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, self.target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=build-manifest"); - - builder.run(&mut cmd); - distdir(builder).join(format!("{}-{}.tar.gz", name, self.target.triple)) + let tarball = Tarball::new(builder, "build-manifest", &self.target.triple); + tarball.add_file(&build_manifest, "bin", 0o755); + tarball.generate() } } diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index ece9bdc7a6499..3b51bf272fcba 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -142,6 +142,7 @@ mod native; mod run; mod sanity; mod setup; +mod tarball; mod test; mod tool; mod toolstate; diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs new file mode 100644 index 0000000000000..728d134434952 --- /dev/null +++ b/src/bootstrap/tarball.rs @@ -0,0 +1,124 @@ +use std::path::{Path, PathBuf}; + +use build_helper::t; + +use crate::builder::Builder; + +#[derive(Copy, Clone)] +pub(crate) enum OverlayKind { + Rust, + LLVM, +} + +impl OverlayKind { + fn included_files(&self) -> &[&str] { + match self { + OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"], + OverlayKind::LLVM => { + &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"] + } + } + } +} + +pub(crate) struct Tarball<'a> { + builder: &'a Builder<'a>, + + pkgname: String, + component: String, + target: String, + overlay: OverlayKind, + + temp_dir: PathBuf, + image_dir: PathBuf, + overlay_dir: PathBuf, + work_dir: PathBuf, +} + +impl<'a> Tarball<'a> { + pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self { + let pkgname = crate::dist::pkgname(builder, component); + + let temp_dir = builder.out.join("tmp").join("tarball").join(component); + let _ = std::fs::remove_dir_all(&temp_dir); + + let image_dir = temp_dir.join("image"); + let overlay_dir = temp_dir.join("overlay"); + let work_dir = temp_dir.join("work"); + + Self { + builder, + + pkgname, + component: component.into(), + target: target.into(), + overlay: OverlayKind::Rust, + + temp_dir, + image_dir, + overlay_dir, + work_dir, + } + } + + pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) { + self.overlay = overlay; + } + + pub(crate) fn image_dir(&self) -> &Path { + t!(std::fs::create_dir_all(&self.image_dir)); + &self.image_dir + } + + pub(crate) fn add_file(&self, src: impl AsRef, destdir: impl AsRef, perms: u32) { + // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply + // uses the base directory as the destination directory. + let destdir = if destdir.as_ref() == Path::new(".") { + self.image_dir.clone() + } else { + self.image_dir.join(destdir.as_ref()) + }; + + t!(std::fs::create_dir_all(&destdir)); + self.builder.install(src.as_ref(), &destdir, perms); + } + + pub(crate) fn add_dir(&self, src: impl AsRef, destdir: impl AsRef) { + t!(std::fs::create_dir_all(destdir.as_ref())); + self.builder.cp_r( + src.as_ref(), + &self.image_dir.join(destdir.as_ref()).join(src.as_ref().file_name().unwrap()), + ); + } + + pub(crate) fn generate(self) -> PathBuf { + t!(std::fs::create_dir_all(&self.overlay_dir)); + self.builder.create(&self.overlay_dir.join("version"), &self.builder.rust_version()); + for file in self.overlay.included_files() { + self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); + } + + let distdir = crate::dist::distdir(self.builder); + let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); + cmd.arg("generate") + .arg("--product-name=Rust") + .arg("--rel-manifest-dir=rustlib") + .arg(format!("--success-message={} installed.", self.component)) + .arg("--image-dir") + .arg(self.image_dir) + .arg("--work-dir") + .arg(self.work_dir) + .arg("--output-dir") + .arg(&distdir) + .arg("--non-installed-overlay") + .arg(self.overlay_dir) + .arg(format!("--package-name={}-{}", self.pkgname, self.target)) + .arg("--legacy-manifest-dirs=rustlib,cargo") + .arg(format!("--component-name={}", self.component)); + self.builder.run(&mut cmd); + + t!(std::fs::remove_dir_all(&self.temp_dir)); + + distdir.join(format!("{}-{}.tar.gz", self.pkgname, self.target)) + } +} From 7be85701cda29bbe715e462be856a61aed5bd4b4 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 26 Nov 2020 17:47:48 +0100 Subject: [PATCH 02/23] bootstrap: convert llvm-tools to use Tarball --- src/bootstrap/dist.rs | 44 +++++++--------------------------------- src/bootstrap/lib.rs | 4 ---- src/bootstrap/tarball.rs | 15 +++++++++++++- 3 files changed, 21 insertions(+), 42 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 0ec896ed2115e..9b0a6e6e19d7a 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2431,56 +2431,26 @@ impl Step for LlvmTools { builder.info(&format!("Dist LlvmTools ({})", target)); let _time = timeit(builder); - let src = builder.src.join("src/llvm-project/llvm"); - let name = pkgname(builder, "llvm-tools"); - let tmp = tmpdir(builder); - let image = tmp.join("llvm-tools-image"); - drop(fs::remove_dir_all(&image)); + let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple); + tarball.set_overlay(OverlayKind::LLVM); + tarball.is_preview(true); // Prepare the image directory let src_bindir = builder.llvm_out(target).join("bin"); - let dst_bindir = image.join("lib/rustlib").join(&*target.triple).join("bin"); - t!(fs::create_dir_all(&dst_bindir)); + let dst_bindir = format!("lib/rustlib/{}/bin", target.triple); for tool in LLVM_TOOLS { let exe = src_bindir.join(exe(tool, target)); - builder.install(&exe, &dst_bindir, 0o755); + tarball.add_file(&exe, &dst_bindir, 0o755); } // Copy libLLVM.so to the target lib dir as well, so the RPATH like // `$ORIGIN/../lib` can find it. It may also be used as a dependency // of `rustc-dev` to support the inherited `-lLLVM` when using the // compiler libraries. - maybe_install_llvm_target(builder, target, &image); - - // Prepare the overlay - let overlay = tmp.join("llvm-tools-overlay"); - drop(fs::remove_dir_all(&overlay)); - builder.create_dir(&overlay); - builder.install(&src.join("README.txt"), &overlay, 0o644); - builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644); - builder.create(&overlay.join("version"), &builder.llvm_tools_vers()); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=llvm-tools-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=llvm-tools-preview"); + maybe_install_llvm_target(builder, target, tarball.image_dir()); - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + Some(tarball.generate()) } } diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 3b51bf272fcba..a47ddfbcc1f18 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -1069,10 +1069,6 @@ impl Build { self.package_vers(&self.version) } - fn llvm_tools_vers(&self) -> String { - self.rust_version() - } - fn llvm_link_tools_dynamically(&self, target: TargetSelection) -> bool { target.contains("linux-gnu") || target.contains("apple-darwin") } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 728d134434952..2c110a7fb24b8 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -33,6 +33,8 @@ pub(crate) struct Tarball<'a> { image_dir: PathBuf, overlay_dir: PathBuf, work_dir: PathBuf, + + is_preview: bool, } impl<'a> Tarball<'a> { @@ -58,6 +60,8 @@ impl<'a> Tarball<'a> { image_dir, overlay_dir, work_dir, + + is_preview: false, } } @@ -65,6 +69,10 @@ impl<'a> Tarball<'a> { self.overlay = overlay; } + pub(crate) fn is_preview(&mut self, is: bool) { + self.is_preview = is; + } + pub(crate) fn image_dir(&self) -> &Path { t!(std::fs::create_dir_all(&self.image_dir)); &self.image_dir @@ -98,6 +106,11 @@ impl<'a> Tarball<'a> { self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); } + let mut component_name = self.component.clone(); + if self.is_preview { + component_name.push_str("-preview"); + } + let distdir = crate::dist::distdir(self.builder); let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); cmd.arg("generate") @@ -114,7 +127,7 @@ impl<'a> Tarball<'a> { .arg(self.overlay_dir) .arg(format!("--package-name={}-{}", self.pkgname, self.target)) .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg(format!("--component-name={}", self.component)); + .arg(format!("--component-name={}", component_name)); self.builder.run(&mut cmd); t!(std::fs::remove_dir_all(&self.temp_dir)); From c768ce138427b1844c1f6594daba9c0e33928032 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 26 Nov 2020 22:20:45 +0100 Subject: [PATCH 03/23] bootstrap: convert rust-docs to use Tarball --- src/bootstrap/dist.rs | 45 ++++++++++------------------------------ src/bootstrap/tarball.rs | 19 ++++++++++------- 2 files changed, 23 insertions(+), 41 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 9b0a6e6e19d7a..26a621c41caed 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -55,7 +55,7 @@ pub struct Docs { } impl Step for Docs { - type Output = PathBuf; + type Output = Option; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -67,13 +67,10 @@ impl Step for Docs { } /// Builds the `rust-docs` installer component. - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option { let host = self.host; - - let name = pkgname(builder, "rust-docs"); - if !builder.config.docs { - return distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)); + return None; } builder.default_doc(None); @@ -81,34 +78,14 @@ impl Step for Docs { builder.info(&format!("Dist docs ({})", host)); let _time = timeit(builder); - let image = tmpdir(builder).join(format!("{}-{}-image", name, host.triple)); - let _ = fs::remove_dir_all(&image); + let dest = "share/doc/rust/html"; - let dst = image.join("share/doc/rust/html"); - t!(fs::create_dir_all(&dst)); - let src = builder.doc_out(host); - builder.cp_r(&src, &dst); - builder.install(&builder.src.join("src/doc/robots.txt"), &dst, 0o644); - - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust-Documentation") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-documentation-is-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, host.triple)) - .arg("--component-name=rust-docs") - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--bulk-dirs=share/doc/rust/html"); - builder.run(&mut cmd); - builder.remove_dir(&image); + let mut tarball = Tarball::new(builder, "rust-docs", &host.triple); + tarball.set_product_name("Rust Documentation"); + tarball.add_dir(&builder.doc_out(host), dest); + tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644); - distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)) + Some(tarball.generate()) } } @@ -1826,7 +1803,7 @@ impl Step for Extended { tarballs.extend(llvm_tools_installer); tarballs.push(analysis_installer); tarballs.push(std_installer); - if builder.config.docs { + if let Some(docs_installer) = docs_installer { tarballs.push(docs_installer); } if target.contains("pc-windows-gnu") { @@ -2509,7 +2486,7 @@ impl Step for RustDev { // Copy the include directory as well; needed mostly to build // librustc_llvm properly (e.g., llvm-config.h is in here). But also // just broadly useful to be able to link against the bundled LLVM. - tarball.add_dir(&builder.llvm_out(target).join("include"), "."); + tarball.add_dir(&builder.llvm_out(target).join("include"), "include"); // Copy libLLVM.so to the target lib dir as well, so the RPATH like // `$ORIGIN/../lib` can find it. It may also be used as a dependency diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 2c110a7fb24b8..50d58d00a66e4 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -27,6 +27,7 @@ pub(crate) struct Tarball<'a> { pkgname: String, component: String, target: String, + product_name: String, overlay: OverlayKind, temp_dir: PathBuf, @@ -54,6 +55,7 @@ impl<'a> Tarball<'a> { pkgname, component: component.into(), target: target.into(), + product_name: "Rust".into(), overlay: OverlayKind::Rust, temp_dir, @@ -69,6 +71,10 @@ impl<'a> Tarball<'a> { self.overlay = overlay; } + pub(crate) fn set_product_name(&mut self, name: &str) { + self.product_name = name.into(); + } + pub(crate) fn is_preview(&mut self, is: bool) { self.is_preview = is; } @@ -91,12 +97,11 @@ impl<'a> Tarball<'a> { self.builder.install(src.as_ref(), &destdir, perms); } - pub(crate) fn add_dir(&self, src: impl AsRef, destdir: impl AsRef) { - t!(std::fs::create_dir_all(destdir.as_ref())); - self.builder.cp_r( - src.as_ref(), - &self.image_dir.join(destdir.as_ref()).join(src.as_ref().file_name().unwrap()), - ); + pub(crate) fn add_dir(&self, src: impl AsRef, dest: impl AsRef) { + let dest = self.image_dir.join(dest.as_ref()); + + t!(std::fs::create_dir_all(&dest)); + self.builder.cp_r(src.as_ref(), &dest); } pub(crate) fn generate(self) -> PathBuf { @@ -114,7 +119,7 @@ impl<'a> Tarball<'a> { let distdir = crate::dist::distdir(self.builder); let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); cmd.arg("generate") - .arg("--product-name=Rust") + .arg(format!("--product-name={}", self.product_name)) .arg("--rel-manifest-dir=rustlib") .arg(format!("--success-message={} installed.", self.component)) .arg("--image-dir") From 8ca46fc7a83734c9622f11f25d16b82316f44bcc Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 26 Nov 2020 22:24:34 +0100 Subject: [PATCH 04/23] bootstrap: convert rustc-docs to use Tarball --- src/bootstrap/dist.rs | 42 ++++++++---------------------------------- 1 file changed, 8 insertions(+), 34 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 26a621c41caed..5a9c82c52a6fd 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -95,7 +95,7 @@ pub struct RustcDocs { } impl Step for RustcDocs { - type Output = PathBuf; + type Output = Option; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -107,47 +107,21 @@ impl Step for RustcDocs { } /// Builds the `rustc-docs` installer component. - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option { let host = self.host; - - let name = pkgname(builder, "rustc-docs"); - if !builder.config.compiler_docs { - return distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)); + return None; } builder.default_doc(None); - - let image = tmpdir(builder).join(format!("{}-{}-image", name, host.triple)); - let _ = fs::remove_dir_all(&image); - - let dst = image.join("share/doc/rust/html/rustc"); - t!(fs::create_dir_all(&dst)); - let src = builder.compiler_doc_out(host); - builder.cp_r(&src, &dst); - - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rustc-Documentation") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rustc-documentation-is-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, host.triple)) - .arg("--component-name=rustc-docs") - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--bulk-dirs=share/doc/rust/html/rustc"); - builder.info(&format!("Dist compiler docs ({})", host)); let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)) + let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple); + tarball.set_product_name("Rustc Documentation"); + tarball.add_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc"); + + Some(tarball.generate()) } } From 82d9eaa54d3a60edc2dd664355e390ec9aa36fa5 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 26 Nov 2020 22:29:33 +0100 Subject: [PATCH 05/23] bootstrap: convert rust-mingw to use Tarball --- src/bootstrap/dist.rs | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 5a9c82c52a6fd..cee8c411fa1fc 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -297,41 +297,23 @@ impl Step for Mingw { /// without any extra installed software (e.g., we bundle gcc, libraries, etc). fn run(self, builder: &Builder<'_>) -> Option { let host = self.host; - if !host.contains("pc-windows-gnu") { return None; } builder.info(&format!("Dist mingw ({})", host)); let _time = timeit(builder); - let name = pkgname(builder, "rust-mingw"); - let image = tmpdir(builder).join(format!("{}-{}-image", name, host.triple)); - let _ = fs::remove_dir_all(&image); - t!(fs::create_dir_all(&image)); + + let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple); + tarball.set_product_name("Rust MinGW"); // The first argument is a "temporary directory" which is just // thrown away (this contains the runtime DLLs included in the rustc package // above) and the second argument is where to place all the MinGW components // (which is what we want). - make_win_dist(&tmpdir(builder), &image, host, &builder); + make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder); - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust-MinGW") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-MinGW-is-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, host.triple)) - .arg("--component-name=rust-mingw") - .arg("--legacy-manifest-dirs=rustlib,cargo"); - builder.run(&mut cmd); - t!(fs::remove_dir_all(&image)); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple))) + Some(tarball.generate()) } } From 0a2e1c5a2c85ff27f2677aa7db1c2deacf34242d Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 26 Nov 2020 22:42:52 +0100 Subject: [PATCH 06/23] bootstrap: convert rustc to use Tarball --- src/bootstrap/dist.rs | 60 ++++++---------------------------------- src/bootstrap/tarball.rs | 3 ++ 2 files changed, 11 insertions(+), 52 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index cee8c411fa1fc..db792886c1665 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -341,30 +341,13 @@ impl Step for Rustc { let compiler = self.compiler; let host = self.compiler.host; - let name = pkgname(builder, "rustc"); - let image = tmpdir(builder).join(format!("{}-{}-image", name, host.triple)); - let _ = fs::remove_dir_all(&image); - let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host.triple)); - let _ = fs::remove_dir_all(&overlay); + builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host.triple)); + let _time = timeit(builder); - // Prepare the rustc "image", what will actually end up getting installed - prepare_image(builder, compiler, &image); + let tarball = Tarball::new(builder, "rustc", &host.triple); - // Prepare the overlay which is part of the tarball but won't actually be - // installed - let cp = |file: &str| { - builder.install(&builder.src.join(file), &overlay, 0o644); - }; - cp("COPYRIGHT"); - cp("LICENSE-APACHE"); - cp("LICENSE-MIT"); - cp("README.md"); - // tiny morsel of metadata is used by rust-packaging - let version = builder.rust_version(); - builder.create(&overlay.join("version"), &version); - if let Some(sha) = builder.rust_sha() { - builder.create(&overlay.join("git-commit-hash"), &sha); - } + // Prepare the rustc "image", what will actually end up getting installed + prepare_image(builder, compiler, tarball.image_dir()); // On MinGW we've got a few runtime DLL dependencies that we need to // include. The first argument to this script is where to put these DLLs @@ -377,38 +360,11 @@ impl Step for Rustc { // install will *also* include the rust-mingw package, which also needs // licenses, so to be safe we just include it here in all MinGW packages. if host.contains("pc-windows-gnu") { - make_win_dist(&image, &tmpdir(builder), host, builder); - - let dst = image.join("share/doc"); - t!(fs::create_dir_all(&dst)); - builder.cp_r(&builder.src.join("src/etc/third-party"), &dst); + make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder); + tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc"); } - // Finally, wrap everything up in a nice tarball! - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-is-ready-to-roll.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, host.triple)) - .arg("--component-name=rustc") - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host.triple)); - let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - builder.remove_dir(&overlay); - - return distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)); + return tarball.generate(); fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) { let host = compiler.host; diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 50d58d00a66e4..294b69c85f488 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -107,6 +107,9 @@ impl<'a> Tarball<'a> { pub(crate) fn generate(self) -> PathBuf { t!(std::fs::create_dir_all(&self.overlay_dir)); self.builder.create(&self.overlay_dir.join("version"), &self.builder.rust_version()); + if let Some(sha) = self.builder.rust_sha() { + self.builder.create(&self.overlay_dir.join("git-commit-hash"), &sha); + } for file in self.overlay.included_files() { self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); } From fd4515cb3f8a8e21d3640ffa8f2b3040a381b87c Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 26 Nov 2020 22:50:38 +0100 Subject: [PATCH 07/23] bootstrap: refactor showing the "dist" info --- src/bootstrap/dist.rs | 21 --------------------- src/bootstrap/tarball.rs | 7 +++++-- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index db792886c1665..0e00649fc0325 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -72,19 +72,14 @@ impl Step for Docs { if !builder.config.docs { return None; } - builder.default_doc(None); - builder.info(&format!("Dist docs ({})", host)); - let _time = timeit(builder); - let dest = "share/doc/rust/html"; let mut tarball = Tarball::new(builder, "rust-docs", &host.triple); tarball.set_product_name("Rust Documentation"); tarball.add_dir(&builder.doc_out(host), dest); tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644); - Some(tarball.generate()) } } @@ -112,15 +107,11 @@ impl Step for RustcDocs { if !builder.config.compiler_docs { return None; } - builder.default_doc(None); - builder.info(&format!("Dist compiler docs ({})", host)); - let _time = timeit(builder); let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple); tarball.set_product_name("Rustc Documentation"); tarball.add_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc"); - Some(tarball.generate()) } } @@ -301,9 +292,6 @@ impl Step for Mingw { return None; } - builder.info(&format!("Dist mingw ({})", host)); - let _time = timeit(builder); - let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple); tarball.set_product_name("Rust MinGW"); @@ -341,9 +329,6 @@ impl Step for Rustc { let compiler = self.compiler; let host = self.compiler.host; - builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host.triple)); - let _time = timeit(builder); - let tarball = Tarball::new(builder, "rustc", &host.triple); // Prepare the rustc "image", what will actually end up getting installed @@ -2318,9 +2303,6 @@ impl Step for LlvmTools { } } - builder.info(&format!("Dist LlvmTools ({})", target)); - let _time = timeit(builder); - let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple); tarball.set_overlay(OverlayKind::LLVM); tarball.is_preview(true); @@ -2375,9 +2357,6 @@ impl Step for RustDev { } } - builder.info(&format!("Dist RustDev ({})", target)); - let _time = timeit(builder); - let mut tarball = Tarball::new(builder, "rust-dev", &target.triple); tarball.set_overlay(OverlayKind::LLVM); diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 294b69c85f488..f99b6f30192dc 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -114,13 +114,17 @@ impl<'a> Tarball<'a> { self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); } + let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); + + self.builder.info(&format!("Dist {} ({})", self.component, self.target)); + let _time = crate::util::timeit(self.builder); + let mut component_name = self.component.clone(); if self.is_preview { component_name.push_str("-preview"); } let distdir = crate::dist::distdir(self.builder); - let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); cmd.arg("generate") .arg(format!("--product-name={}", self.product_name)) .arg("--rel-manifest-dir=rustlib") @@ -137,7 +141,6 @@ impl<'a> Tarball<'a> { .arg("--legacy-manifest-dirs=rustlib,cargo") .arg(format!("--component-name={}", component_name)); self.builder.run(&mut cmd); - t!(std::fs::remove_dir_all(&self.temp_dir)); distdir.join(format!("{}-{}.tar.gz", self.pkgname, self.target)) From 79f60fbd0410d1cfcbc4ddc8cba7bb11e1dd65ba Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 26 Nov 2020 22:54:35 +0100 Subject: [PATCH 08/23] bootstrap: convert rust-std to use Tarball --- src/bootstrap/dist.rs | 38 ++++++++------------------------------ src/bootstrap/tarball.rs | 10 ++++++++++ 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 0e00649fc0325..98f2b28918a17 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -559,7 +559,7 @@ pub struct Std { } impl Step for Std { - type Output = PathBuf; + type Output = Option; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -577,46 +577,24 @@ impl Step for Std { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option { let compiler = self.compiler; let target = self.target; - let name = pkgname(builder, "rust-std"); - let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)); if skip_host_target_lib(builder, compiler) { - return archive; + return None; } builder.ensure(compile::Std { compiler, target }); - let image = tmpdir(builder).join(format!("{}-{}-image", name, target.triple)); - let _ = fs::remove_dir_all(&image); + let mut tarball = Tarball::new(builder, "rust-std", &target.triple); + tarball.include_target_in_component_name(true); let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); let stamp = compile::libstd_stamp(builder, compiler_to_use, target); - copy_target_libs(builder, target, &image, &stamp); - - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=std-is-standing-at-the-ready.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg(format!("--component-name=rust-std-{}", target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo"); + copy_target_libs(builder, target, &tarball.image_dir(), &stamp); - builder - .info(&format!("Dist std stage{} ({} -> {})", compiler.stage, &compiler.host, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - archive + Some(tarball.generate()) } } @@ -1699,7 +1677,7 @@ impl Step for Extended { tarballs.extend(rustfmt_installer.clone()); tarballs.extend(llvm_tools_installer); tarballs.push(analysis_installer); - tarballs.push(std_installer); + tarballs.push(std_installer.expect("missing std")); if let Some(docs_installer) = docs_installer { tarballs.push(docs_installer); } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index f99b6f30192dc..bde437723bba1 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -35,6 +35,7 @@ pub(crate) struct Tarball<'a> { overlay_dir: PathBuf, work_dir: PathBuf, + include_target_in_component_name: bool, is_preview: bool, } @@ -63,6 +64,7 @@ impl<'a> Tarball<'a> { overlay_dir, work_dir, + include_target_in_component_name: false, is_preview: false, } } @@ -75,6 +77,10 @@ impl<'a> Tarball<'a> { self.product_name = name.into(); } + pub(crate) fn include_target_in_component_name(&mut self, include: bool) { + self.include_target_in_component_name = include; + } + pub(crate) fn is_preview(&mut self, is: bool) { self.is_preview = is; } @@ -123,6 +129,10 @@ impl<'a> Tarball<'a> { if self.is_preview { component_name.push_str("-preview"); } + if self.include_target_in_component_name { + component_name.push('-'); + component_name.push_str(&self.target); + } let distdir = crate::dist::distdir(self.builder); cmd.arg("generate") From c0cadc9eb72e6147647bd1b7995851e371167f24 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 27 Nov 2020 12:46:02 +0100 Subject: [PATCH 09/23] bootstrap: convert rustc-dev to use Tarball --- src/bootstrap/dist.rs | 56 +++++++++++++------------------------------ 1 file changed, 16 insertions(+), 40 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 98f2b28918a17..1e0f3b9695843 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -605,7 +605,7 @@ pub struct RustcDev { } impl Step for RustcDev { - type Output = PathBuf; + type Output = Option; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -624,60 +624,36 @@ impl Step for RustcDev { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option { let compiler = self.compiler; let target = self.target; - - let name = pkgname(builder, "rustc-dev"); - let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)); if skip_host_target_lib(builder, compiler) { - return archive; + return None; } builder.ensure(compile::Rustc { compiler, target }); - let image = tmpdir(builder).join(format!("{}-{}-image", name, target.triple)); - let _ = fs::remove_dir_all(&image); + let tarball = Tarball::new(builder, "rustc-dev", &target.triple); let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); let stamp = compile::librustc_stamp(builder, compiler_to_use, target); - copy_target_libs(builder, target, &image, &stamp); + copy_target_libs(builder, target, tarball.image_dir(), &stamp); - // Copy compiler sources. - let dst_src = image.join("lib/rustlib/rustc-src/rust"); - t!(fs::create_dir_all(&dst_src)); - - let src_files = ["Cargo.lock"]; + let src_files = &["Cargo.lock"]; // This is the reduced set of paths which will become the rustc-dev component // (essentially the compiler crates and all of their path dependencies). - copy_src_dirs(builder, &builder.src, &["compiler"], &[], &dst_src); - for file in src_files.iter() { - builder.copy(&builder.src.join(file), &dst_src.join(file)); + copy_src_dirs( + builder, + &builder.src, + &["compiler"], + &[], + &tarball.image_dir().join("lib/rustlib/rustc-src/rust"), + ); + for file in src_files { + tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644); } - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-is-ready-to-develop.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg(format!("--component-name=rustc-dev-{}", target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder.info(&format!( - "Dist rustc-dev stage{} ({} -> {})", - compiler.stage, &compiler.host, target - )); - let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - archive + Some(tarball.generate()) } } From c4aaff65f0cac8fe4375423f36d544440e47878b Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 27 Nov 2020 12:52:44 +0100 Subject: [PATCH 10/23] bootstrap: convert rust-analysis to use Tarball --- src/bootstrap/dist.rs | 48 +++++++++++-------------------------------- 1 file changed, 12 insertions(+), 36 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 1e0f3b9695843..e342c0ddf639b 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -664,7 +664,7 @@ pub struct Analysis { } impl Step for Analysis { - type Output = PathBuf; + type Output = Option; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -687,52 +687,26 @@ impl Step for Analysis { } /// Creates a tarball of save-analysis metadata, if available. - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option { let compiler = self.compiler; let target = self.target; assert!(builder.config.extended); - let name = pkgname(builder, "rust-analysis"); - if compiler.host != builder.config.build { - return distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)); + return None; } builder.ensure(compile::Std { compiler, target }); - - let image = tmpdir(builder).join(format!("{}-{}-image", name, target.triple)); - let src = builder .stage_out(compiler, Mode::Std) .join(target.triple) .join(builder.cargo_dir()) - .join("deps"); - - let image_src = src.join("save-analysis"); - let dst = image.join("lib/rustlib").join(target.triple).join("analysis"); - t!(fs::create_dir_all(&dst)); - builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst)); - builder.cp_r(&image_src, &dst); - - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=save-analysis-saved.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg(format!("--component-name=rust-analysis-{}", target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo"); + .join("deps") + .join("save-analysis"); - builder.info("Dist analysis"); - let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)) + let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple); + tarball.include_target_in_component_name(true); + tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple)); + Some(tarball.generate()) } } @@ -1652,7 +1626,9 @@ impl Step for Extended { tarballs.extend(miri_installer.clone()); tarballs.extend(rustfmt_installer.clone()); tarballs.extend(llvm_tools_installer); - tarballs.push(analysis_installer); + if let Some(analysis_installer) = analysis_installer { + tarballs.push(analysis_installer); + } tarballs.push(std_installer.expect("missing std")); if let Some(docs_installer) = docs_installer { tarballs.push(docs_installer); From 8a711a00a7f0cd5470255a65a711d47e46d46c14 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 27 Nov 2020 13:11:11 +0100 Subject: [PATCH 11/23] bootstrap: convert cargo to use Tarball --- src/bootstrap/dist.rs | 71 +++++++++------------------------------- src/bootstrap/tarball.rs | 30 ++++++++++++++++- 2 files changed, 44 insertions(+), 57 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index e342c0ddf639b..7e7c7edceff06 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1015,72 +1015,31 @@ impl Step for Cargo { let compiler = self.compiler; let target = self.target; + let cargo = builder.ensure(tool::Cargo { compiler, target }); let src = builder.src.join("src/tools/cargo"); let etc = src.join("src/etc"); - let release_num = builder.release_num("cargo"); - let name = pkgname(builder, "cargo"); - let version = builder.cargo_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("cargo-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); // Prepare the image directory - builder.create_dir(&image.join("share/zsh/site-functions")); - builder.create_dir(&image.join("etc/bash_completion.d")); - let cargo = builder.ensure(tool::Cargo { compiler, target }); - builder.install(&cargo, &image.join("bin"), 0o755); + let mut tarball = Tarball::new(builder, "cargo", &target.triple); + tarball.set_overlay(OverlayKind::Cargo); + + tarball.add_file(&cargo, "bin", 0o755); + tarball.add_file(src.join("README.md"), "share/doc/cargo", 0o644); + tarball.add_file(src.join("LICENSE-MIT"), "share/doc/cargo", 0o644); + tarball.add_file(src.join("LICENSE-APACHE"), "share/doc/cargo", 0o644); + tarball.add_file(src.join("LICENSE-THIRD-PARTY"), "share/doc/cargo", 0o644); + tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644); + tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo"); + tarball.add_dir(etc.join("man"), "share/man/man1"); + for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") { let dirent = dirent.expect("read dir entry"); if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") { - builder.install(&dirent.path(), &image.join("libexec"), 0o755); + tarball.add_file(&dirent.path(), "libexec", 0o755); } } - for man in t!(etc.join("man").read_dir()) { - let man = t!(man); - builder.install(&man.path(), &image.join("share/man/man1"), 0o644); - } - builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644); - builder.copy(&etc.join("cargo.bashcomp.sh"), &image.join("etc/bash_completion.d/cargo")); - let doc = image.join("share/doc/cargo"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("cargo-overlay"); - drop(fs::remove_dir_all(&overlay)); - builder.create_dir(&overlay); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644); - builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-is-ready-to-roll.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--component-name=cargo") - .arg("--legacy-manifest-dirs=rustlib,cargo"); - builder.info(&format!("Dist cargo stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)) + tarball.generate() } } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index bde437723bba1..5340995ce96f4 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -8,6 +8,7 @@ use crate::builder::Builder; pub(crate) enum OverlayKind { Rust, LLVM, + Cargo, } impl OverlayKind { @@ -17,6 +18,22 @@ impl OverlayKind { OverlayKind::LLVM => { &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"] } + OverlayKind::Cargo => &[ + "src/tools/cargo/README.md", + "src/tools/cargo/LICENSE-MIT", + "src/tools/cargo/LICENSE-APACHE", + "src/tools/cargo/LICENSE-THIRD-PARTY", + ], + } + } + + fn version(&self, builder: &Builder<'_>) -> String { + match self { + OverlayKind::Rust => builder.rust_version(), + OverlayKind::LLVM => builder.rust_version(), + OverlayKind::Cargo => { + builder.cargo_info.version(builder, &builder.release_num("cargo")) + } } } } @@ -103,6 +120,17 @@ impl<'a> Tarball<'a> { self.builder.install(src.as_ref(), &destdir, perms); } + pub(crate) fn add_renamed_file( + &self, + src: impl AsRef, + destdir: impl AsRef, + new_name: &str, + ) { + let destdir = self.image_dir.join(destdir.as_ref()); + t!(std::fs::create_dir_all(&destdir)); + self.builder.copy(src.as_ref(), &destdir.join(new_name)); + } + pub(crate) fn add_dir(&self, src: impl AsRef, dest: impl AsRef) { let dest = self.image_dir.join(dest.as_ref()); @@ -112,7 +140,7 @@ impl<'a> Tarball<'a> { pub(crate) fn generate(self) -> PathBuf { t!(std::fs::create_dir_all(&self.overlay_dir)); - self.builder.create(&self.overlay_dir.join("version"), &self.builder.rust_version()); + self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder)); if let Some(sha) = self.builder.rust_sha() { self.builder.create(&self.overlay_dir.join("git-commit-hash"), &sha); } From 521b884913b603ddbdcc1af97b772e40d4ea5f84 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 27 Nov 2020 17:43:03 +0100 Subject: [PATCH 12/23] bootstrap: convert clippy to use Tarball --- src/bootstrap/dist.rs | 58 +++++++--------------------------------- src/bootstrap/tarball.rs | 9 +++++++ 2 files changed, 19 insertions(+), 48 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 7e7c7edceff06..47ad8fae26b1d 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1256,16 +1256,6 @@ impl Step for Clippy { let target = self.target; assert!(builder.config.extended); - let src = builder.src.join("src/tools/clippy"); - let release_num = builder.release_num("clippy"); - let name = pkgname(builder, "clippy"); - let version = builder.clippy_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("clippy-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); - // Prepare the image directory // We expect clippy to build, because we've exited this step above if tool // state for clippy isn't testing. @@ -1275,45 +1265,17 @@ impl Step for Clippy { let cargoclippy = builder .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() }) .expect("clippy expected to build - essential tool"); + let src = builder.src.join("src/tools/clippy"); - builder.install(&clippy, &image.join("bin"), 0o755); - builder.install(&cargoclippy, &image.join("bin"), 0o755); - let doc = image.join("share/doc/clippy"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("clippy-overlay"); - drop(fs::remove_dir_all(&overlay)); - t!(fs::create_dir_all(&overlay)); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=clippy-ready-to-serve.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=clippy-preview"); - - builder.info(&format!("Dist clippy stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)) + let mut tarball = Tarball::new(builder, "clippy", &target.triple); + tarball.set_overlay(OverlayKind::Clippy); + tarball.is_preview(true); + tarball.add_file(clippy, "bin", 0o755); + tarball.add_file(cargoclippy, "bin", 0o755); + tarball.add_file(src.join("README.md"), "share/doc/clippy", 0o644); + tarball.add_file(src.join("LICENSE-APACHE"), "share/doc/clippy", 0o644); + tarball.add_file(src.join("LICENSE-MIT"), "share/doc/clippy", 0o644); + tarball.generate() } } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 5340995ce96f4..c0c84222e7dd8 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -9,6 +9,7 @@ pub(crate) enum OverlayKind { Rust, LLVM, Cargo, + Clippy, } impl OverlayKind { @@ -24,6 +25,11 @@ impl OverlayKind { "src/tools/cargo/LICENSE-APACHE", "src/tools/cargo/LICENSE-THIRD-PARTY", ], + OverlayKind::Clippy => &[ + "src/tools/clippy/README.md", + "src/tools/clippy/LICENSE-APACHE", + "src/tools/clippy/LICENSE-MIT", + ], } } @@ -34,6 +40,9 @@ impl OverlayKind { OverlayKind::Cargo => { builder.cargo_info.version(builder, &builder.release_num("cargo")) } + OverlayKind::Clippy => { + builder.clippy_info.version(builder, &builder.release_num("clippy")) + } } } } From 32afb3c436eb3909a636fd4fce20a1ba03cc88ec Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 27 Nov 2020 17:56:22 +0100 Subject: [PATCH 13/23] bootstrap: simplify including licenses and readmes to tarballs --- src/bootstrap/dist.rs | 10 ++-------- src/bootstrap/tarball.rs | 10 ++++++++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 47ad8fae26b1d..aa4d518d0143a 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1024,13 +1024,10 @@ impl Step for Cargo { tarball.set_overlay(OverlayKind::Cargo); tarball.add_file(&cargo, "bin", 0o755); - tarball.add_file(src.join("README.md"), "share/doc/cargo", 0o644); - tarball.add_file(src.join("LICENSE-MIT"), "share/doc/cargo", 0o644); - tarball.add_file(src.join("LICENSE-APACHE"), "share/doc/cargo", 0o644); - tarball.add_file(src.join("LICENSE-THIRD-PARTY"), "share/doc/cargo", 0o644); tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644); tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo"); tarball.add_dir(etc.join("man"), "share/man/man1"); + tarball.add_legal_and_readme_to("share/doc/cargo"); for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") { let dirent = dirent.expect("read dir entry"); @@ -1265,16 +1262,13 @@ impl Step for Clippy { let cargoclippy = builder .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() }) .expect("clippy expected to build - essential tool"); - let src = builder.src.join("src/tools/clippy"); let mut tarball = Tarball::new(builder, "clippy", &target.triple); tarball.set_overlay(OverlayKind::Clippy); tarball.is_preview(true); tarball.add_file(clippy, "bin", 0o755); tarball.add_file(cargoclippy, "bin", 0o755); - tarball.add_file(src.join("README.md"), "share/doc/clippy", 0o644); - tarball.add_file(src.join("LICENSE-APACHE"), "share/doc/clippy", 0o644); - tarball.add_file(src.join("LICENSE-MIT"), "share/doc/clippy", 0o644); + tarball.add_legal_and_readme_to("share/doc/clippy"); tarball.generate() } } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index c0c84222e7dd8..2c8f69e94b3ff 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -13,7 +13,7 @@ pub(crate) enum OverlayKind { } impl OverlayKind { - fn included_files(&self) -> &[&str] { + fn legal_and_readme(&self) -> &[&str] { match self { OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"], OverlayKind::LLVM => { @@ -140,6 +140,12 @@ impl<'a> Tarball<'a> { self.builder.copy(src.as_ref(), &destdir.join(new_name)); } + pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef) { + for file in self.overlay.legal_and_readme() { + self.add_file(self.builder.src.join(file), destdir.as_ref(), 0o644); + } + } + pub(crate) fn add_dir(&self, src: impl AsRef, dest: impl AsRef) { let dest = self.image_dir.join(dest.as_ref()); @@ -153,7 +159,7 @@ impl<'a> Tarball<'a> { if let Some(sha) = self.builder.rust_sha() { self.builder.create(&self.overlay_dir.join("git-commit-hash"), &sha); } - for file in self.overlay.included_files() { + for file in self.overlay.legal_and_readme() { self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); } From 2073ea5c07ea5a83d17fb7d08f67cc55e1010aaf Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 27 Nov 2020 18:03:41 +0100 Subject: [PATCH 14/23] bootstrap: convert miri to use Tarball --- src/bootstrap/dist.rs | 58 +++++----------------------------------- src/bootstrap/tarball.rs | 7 +++++ 2 files changed, 14 insertions(+), 51 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index aa4d518d0143a..a884e7ee2cc4a 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1303,19 +1303,6 @@ impl Step for Miri { let target = self.target; assert!(builder.config.extended); - let src = builder.src.join("src/tools/miri"); - let release_num = builder.release_num("miri"); - let name = pkgname(builder, "miri"); - let version = builder.miri_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("miri-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); - - // Prepare the image directory - // We expect miri to build, because we've exited this step above if tool - // state for miri isn't testing. let miri = builder .ensure(tool::Miri { compiler, target, extra_features: Vec::new() }) .or_else(|| { @@ -1329,44 +1316,13 @@ impl Step for Miri { None })?; - builder.install(&miri, &image.join("bin"), 0o755); - builder.install(&cargomiri, &image.join("bin"), 0o755); - let doc = image.join("share/doc/miri"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("miri-overlay"); - drop(fs::remove_dir_all(&overlay)); - t!(fs::create_dir_all(&overlay)); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=miri-ready-to-serve.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=miri-preview"); - - builder.info(&format!("Dist miri stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + let mut tarball = Tarball::new(builder, "miri", &target.triple); + tarball.set_overlay(OverlayKind::Miri); + tarball.is_preview(true); + tarball.add_file(miri, "bin", 0o755); + tarball.add_file(cargomiri, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/miri"); + Some(tarball.generate()) } } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 2c8f69e94b3ff..0e705a93902e8 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -10,6 +10,7 @@ pub(crate) enum OverlayKind { LLVM, Cargo, Clippy, + Miri, } impl OverlayKind { @@ -30,6 +31,11 @@ impl OverlayKind { "src/tools/clippy/LICENSE-APACHE", "src/tools/clippy/LICENSE-MIT", ], + OverlayKind::Miri => &[ + "src/tools/miri/README.md", + "src/tools/miri/LICENSE-APACHE", + "src/tools/miri/LICENSE-MIT", + ], } } @@ -43,6 +49,7 @@ impl OverlayKind { OverlayKind::Clippy => { builder.clippy_info.version(builder, &builder.release_num("clippy")) } + OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")), } } } From 2b54f0de3794e5fc38f0d4eb1364cb67a041aa4f Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 27 Nov 2020 18:07:58 +0100 Subject: [PATCH 15/23] bootstrap: convert rustfmt to use Tarball --- src/bootstrap/dist.rs | 56 +++++----------------------------------- src/bootstrap/tarball.rs | 9 +++++++ 2 files changed, 16 insertions(+), 49 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index a884e7ee2cc4a..35e0aeba14652 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1355,17 +1355,6 @@ impl Step for Rustfmt { let compiler = self.compiler; let target = self.target; - let src = builder.src.join("src/tools/rustfmt"); - let release_num = builder.release_num("rustfmt"); - let name = pkgname(builder, "rustfmt"); - let version = builder.rustfmt_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("rustfmt-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); - - // Prepare the image directory let rustfmt = builder .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() }) .or_else(|| { @@ -1379,44 +1368,13 @@ impl Step for Rustfmt { None })?; - builder.install(&rustfmt, &image.join("bin"), 0o755); - builder.install(&cargofmt, &image.join("bin"), 0o755); - let doc = image.join("share/doc/rustfmt"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("rustfmt-overlay"); - drop(fs::remove_dir_all(&overlay)); - builder.create_dir(&overlay); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=rustfmt-ready-to-fmt.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=rustfmt-preview"); - - builder.info(&format!("Dist Rustfmt stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + let mut tarball = Tarball::new(builder, "rustfmt", &target.triple); + tarball.set_overlay(OverlayKind::Rustfmt); + tarball.is_preview(true); + tarball.add_file(rustfmt, "bin", 0o755); + tarball.add_file(cargofmt, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rustfmt"); + Some(tarball.generate()) } } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 0e705a93902e8..8543f9d4a386d 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -11,6 +11,7 @@ pub(crate) enum OverlayKind { Cargo, Clippy, Miri, + Rustfmt, } impl OverlayKind { @@ -36,6 +37,11 @@ impl OverlayKind { "src/tools/miri/LICENSE-APACHE", "src/tools/miri/LICENSE-MIT", ], + OverlayKind::Rustfmt => &[ + "src/tools/rustfmt/README.md", + "src/tools/rustfmt/LICENSE-APACHE", + "src/tools/rustfmt/LICENSE-MIT", + ], } } @@ -50,6 +56,9 @@ impl OverlayKind { builder.clippy_info.version(builder, &builder.release_num("clippy")) } OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")), + OverlayKind::Rustfmt => { + builder.rustfmt_info.version(builder, &builder.release_num("rustfmt")) + } } } } From cfb23e845e56718fa33b7b76d30e2edec992d659 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 27 Nov 2020 18:12:17 +0100 Subject: [PATCH 16/23] bootstrap: convert rls to use Tarball --- src/bootstrap/dist.rs | 56 +++++----------------------------------- src/bootstrap/tarball.rs | 7 +++++ 2 files changed, 13 insertions(+), 50 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 35e0aeba14652..c31a938c746fd 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1070,19 +1070,6 @@ impl Step for Rls { let target = self.target; assert!(builder.config.extended); - let src = builder.src.join("src/tools/rls"); - let release_num = builder.release_num("rls"); - let name = pkgname(builder, "rls"); - let version = builder.rls_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("rls-image"); - drop(fs::remove_dir_all(&image)); - t!(fs::create_dir_all(&image)); - - // Prepare the image directory - // We expect RLS to build, because we've exited this step above if tool - // state for RLS isn't testing. let rls = builder .ensure(tool::Rls { compiler, target, extra_features: Vec::new() }) .or_else(|| { @@ -1090,43 +1077,12 @@ impl Step for Rls { None })?; - builder.install(&rls, &image.join("bin"), 0o755); - let doc = image.join("share/doc/rls"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("rls-overlay"); - drop(fs::remove_dir_all(&overlay)); - t!(fs::create_dir_all(&overlay)); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=RLS-ready-to-serve.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=rls-preview"); - - builder.info(&format!("Dist RLS stage{} ({})", compiler.stage, target.triple)); - let _time = timeit(builder); - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + let mut tarball = Tarball::new(builder, "rls", &target.triple); + tarball.set_overlay(OverlayKind::RLS); + tarball.is_preview(true); + tarball.add_file(rls, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rls"); + Some(tarball.generate()) } } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 8543f9d4a386d..efc85a1445aa2 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -12,6 +12,7 @@ pub(crate) enum OverlayKind { Clippy, Miri, Rustfmt, + RLS, } impl OverlayKind { @@ -42,6 +43,11 @@ impl OverlayKind { "src/tools/rustfmt/LICENSE-APACHE", "src/tools/rustfmt/LICENSE-MIT", ], + OverlayKind::RLS => &[ + "src/tools/rls/README.md", + "src/tools/rls/LICENSE-APACHE", + "src/tools/rls/LICENSE-MIT", + ], } } @@ -59,6 +65,7 @@ impl OverlayKind { OverlayKind::Rustfmt => { builder.rustfmt_info.version(builder, &builder.release_num("rustfmt")) } + OverlayKind::RLS => builder.rls_info.version(builder, &builder.release_num("rls")), } } } From 2e0a16cf0d67f61d85ff7631846e9c3c6e20c85a Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 27 Nov 2020 18:20:23 +0100 Subject: [PATCH 17/23] bootstrap: convert rust-analyzer to use Tarball --- src/bootstrap/dist.rs | 56 +++++----------------------------------- src/bootstrap/tarball.rs | 9 +++++++ 2 files changed, 15 insertions(+), 50 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index c31a938c746fd..b1a61500a70d9 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1122,60 +1122,16 @@ impl Step for RustAnalyzer { return None; } - let src = builder.src.join("src/tools/rust-analyzer"); - let release_num = builder.release_num("rust-analyzer/crates/rust-analyzer"); - let name = pkgname(builder, "rust-analyzer"); - let version = builder.rust_analyzer_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("rust-analyzer-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); - - // Prepare the image directory - // We expect rust-analyer to always build, as it doesn't depend on rustc internals - // and doesn't have associated toolstate. let rust_analyzer = builder .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() }) .expect("rust-analyzer always builds"); - builder.install(&rust_analyzer, &image.join("bin"), 0o755); - let doc = image.join("share/doc/rust-analyzer"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("rust-analyzer-overlay"); - drop(fs::remove_dir_all(&overlay)); - t!(fs::create_dir_all(&overlay)); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=rust-analyzer-ready-to-serve.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=rust-analyzer-preview"); - - builder.info(&format!("Dist rust-analyzer stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); + tarball.set_overlay(OverlayKind::RustAnalyzer); + tarball.is_preview(true); + tarball.add_file(rust_analyzer, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rust-analyzer"); + Some(tarball.generate()) } } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index efc85a1445aa2..8a23d36346e3b 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -13,6 +13,7 @@ pub(crate) enum OverlayKind { Miri, Rustfmt, RLS, + RustAnalyzer, } impl OverlayKind { @@ -48,6 +49,11 @@ impl OverlayKind { "src/tools/rls/LICENSE-APACHE", "src/tools/rls/LICENSE-MIT", ], + OverlayKind::RustAnalyzer => &[ + "src/tools/rust-analyzer/README.md", + "src/tools/rust-analyzer/LICENSE-APACHE", + "src/tools/rust-analyzer/LICENSE-MIT", + ], } } @@ -66,6 +72,9 @@ impl OverlayKind { builder.rustfmt_info.version(builder, &builder.release_num("rustfmt")) } OverlayKind::RLS => builder.rls_info.version(builder, &builder.release_num("rls")), + OverlayKind::RustAnalyzer => builder + .rust_analyzer_info + .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")), } } } From 1906c42962b1f2bef084474e09b211e48ed2bda7 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 30 Nov 2020 13:25:34 +0100 Subject: [PATCH 18/23] bootstrap: convert rust-src to use Tarball --- src/bootstrap/dist.rs | 30 +++--------------------------- src/bootstrap/tarball.rs | 31 +++++++++++++++++++++++++------ 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index b1a61500a70d9..c89b378f82060 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -814,9 +814,7 @@ impl Step for Src { /// Creates the `rust-src` installer component fn run(self, builder: &Builder<'_>) -> PathBuf { - let name = pkgname(builder, "rust-src"); - let image = tmpdir(builder).join(format!("{}-image", name)); - let _ = fs::remove_dir_all(&image); + let tarball = Tarball::new_targetless(builder, "rust-src"); // A lot of tools expect the rust-src component to be entirely in this directory, so if you // change that (e.g. by adding another directory `lib/rustlib/src/foo` or @@ -825,8 +823,7 @@ impl Step for Src { // // NOTE: if you update the paths here, you also should update the "virtual" path // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs` - let dst_src = image.join("lib/rustlib/src/rust"); - t!(fs::create_dir_all(&dst_src)); + let dst_src = tarball.image_dir().join("lib/rustlib/src/rust"); let src_files = ["Cargo.lock"]; // This is the reduced set of paths which will become the rust-src component @@ -846,28 +843,7 @@ impl Step for Src { builder.copy(&builder.src.join(file), &dst_src.join(file)); } - // Create source tarball in rust-installer format - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Awesome-Source.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}", name)) - .arg("--component-name=rust-src") - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder.info("Dist src"); - let _time = timeit(builder); - builder.run(&mut cmd); - - builder.remove_dir(&image); - distdir(builder).join(&format!("{}.tar.gz", name)) + tarball.generate() } } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 8a23d36346e3b..27769cab5af09 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -84,7 +84,7 @@ pub(crate) struct Tarball<'a> { pkgname: String, component: String, - target: String, + target: Option, product_name: String, overlay: OverlayKind, @@ -99,6 +99,14 @@ pub(crate) struct Tarball<'a> { impl<'a> Tarball<'a> { pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self { + Self::new_inner(builder, component, Some(target.into())) + } + + pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self { + Self::new_inner(builder, component, None) + } + + fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option) -> Self { let pkgname = crate::dist::pkgname(builder, component); let temp_dir = builder.out.join("tmp").join("tarball").join(component); @@ -113,7 +121,7 @@ impl<'a> Tarball<'a> { pkgname, component: component.into(), - target: target.into(), + target, product_name: "Rust".into(), overlay: OverlayKind::Rust, @@ -197,7 +205,14 @@ impl<'a> Tarball<'a> { let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); - self.builder.info(&format!("Dist {} ({})", self.component, self.target)); + let package_name = if let Some(target) = &self.target { + self.builder.info(&format!("Dist {} ({})", self.component, target)); + format!("{}-{}", self.pkgname, target) + } else { + self.builder.info(&format!("Dist {}", self.component)); + self.pkgname.clone() + }; + let _time = crate::util::timeit(self.builder); let mut component_name = self.component.clone(); @@ -206,7 +221,11 @@ impl<'a> Tarball<'a> { } if self.include_target_in_component_name { component_name.push('-'); - component_name.push_str(&self.target); + component_name.push_str( + &self + .target + .expect("include_target_in_component_name used in a targetless tarball"), + ); } let distdir = crate::dist::distdir(self.builder); @@ -222,12 +241,12 @@ impl<'a> Tarball<'a> { .arg(&distdir) .arg("--non-installed-overlay") .arg(self.overlay_dir) - .arg(format!("--package-name={}-{}", self.pkgname, self.target)) + .arg(format!("--package-name={}", package_name)) .arg("--legacy-manifest-dirs=rustlib,cargo") .arg(format!("--component-name={}", component_name)); self.builder.run(&mut cmd); t!(std::fs::remove_dir_all(&self.temp_dir)); - distdir.join(format!("{}-{}.tar.gz", self.pkgname, self.target)) + distdir.join(format!("{}.tar.gz", package_name)) } } From 48924ab7088802123a64af77e5201ddfc1f1a733 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 2 Dec 2020 13:29:45 +0100 Subject: [PATCH 19/23] bootstrap: convert rust to use Tarball --- src/bootstrap/dist.rs | 43 +++----------------- src/bootstrap/tarball.rs | 85 ++++++++++++++++++++++++++++------------ 2 files changed, 65 insertions(+), 63 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index c89b378f82060..a68f139881744 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1314,21 +1314,7 @@ impl Step for Extended { let std_installer = builder.ensure(Std { compiler: builder.compiler(stage, target), target }); - let tmp = tmpdir(builder); - let overlay = tmp.join("extended-overlay"); let etc = builder.src.join("src/etc/installer"); - let work = tmp.join("work"); - - let _ = fs::remove_dir_all(&overlay); - builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644); - builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644); - builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644); - let version = builder.rust_version(); - builder.create(&overlay.join("version"), &version); - if let Some(sha) = builder.rust_sha() { - builder.create(&overlay.join("git-commit-hash"), &sha); - } - builder.install(&etc.join("README.md"), &overlay, 0o644); // When rust-std package split from rustc, we needed to ensure that during // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering @@ -1353,31 +1339,12 @@ impl Step for Extended { if target.contains("pc-windows-gnu") { tarballs.push(mingw_installer.unwrap()); } - let mut input_tarballs = tarballs[0].as_os_str().to_owned(); - for tarball in &tarballs[1..] { - input_tarballs.push(","); - input_tarballs.push(tarball); - } - builder.info("building combined installer"); - let mut cmd = rust_installer(builder); - cmd.arg("combine") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-is-ready-to-roll.") - .arg("--work-dir") - .arg(&work) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--input-tarballs") - .arg(input_tarballs) - .arg("--non-installed-overlay") - .arg(&overlay); - let time = timeit(&builder); - builder.run(&mut cmd); - drop(time); + let mut tarball = Tarball::new(builder, "rust", &target.triple); + let work = tarball.persist_work_dir(); + tarball.combine(&tarballs); + + let tmp = tmpdir(builder).join("combined-tarball"); let mut license = String::new(); license += &builder.read(&builder.src.join("COPYRIGHT")); diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 27769cab5af09..b4146450596d5 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -1,4 +1,7 @@ -use std::path::{Path, PathBuf}; +use std::{ + path::{Path, PathBuf}, + process::Command, +}; use build_helper::t; @@ -95,6 +98,7 @@ pub(crate) struct Tarball<'a> { include_target_in_component_name: bool, is_preview: bool, + delete_temp_dir: bool, } impl<'a> Tarball<'a> { @@ -132,6 +136,7 @@ impl<'a> Tarball<'a> { include_target_in_component_name: false, is_preview: false, + delete_temp_dir: true, } } @@ -193,7 +198,53 @@ impl<'a> Tarball<'a> { self.builder.cp_r(src.as_ref(), &dest); } + pub(crate) fn persist_work_dir(&mut self) -> PathBuf { + self.delete_temp_dir = false; + self.work_dir.clone() + } + pub(crate) fn generate(self) -> PathBuf { + let mut component_name = self.component.clone(); + if self.is_preview { + component_name.push_str("-preview"); + } + if self.include_target_in_component_name { + component_name.push('-'); + component_name.push_str( + &self + .target + .as_ref() + .expect("include_target_in_component_name used in a targetless tarball"), + ); + } + + self.run(|this, cmd| { + cmd.arg("generate") + .arg("--image-dir") + .arg(&this.image_dir) + .arg("--non-installed-overlay") + .arg(&this.overlay_dir) + .arg(format!("--component-name={}", &component_name)); + }) + } + + pub(crate) fn combine(self, tarballs: &[PathBuf]) { + let mut input_tarballs = tarballs[0].as_os_str().to_os_string(); + for tarball in &tarballs[1..] { + input_tarballs.push(","); + input_tarballs.push(tarball); + } + + self.run(|this, cmd| { + cmd.arg("combine") + .arg("--input-tarballs") + .arg(input_tarballs) + .arg("--non-installed-overlay") + .arg(&this.overlay_dir); + }); + } + + fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> PathBuf { t!(std::fs::create_dir_all(&self.overlay_dir)); self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder)); if let Some(sha) = self.builder.rust_sha() { @@ -215,37 +266,21 @@ impl<'a> Tarball<'a> { let _time = crate::util::timeit(self.builder); - let mut component_name = self.component.clone(); - if self.is_preview { - component_name.push_str("-preview"); - } - if self.include_target_in_component_name { - component_name.push('-'); - component_name.push_str( - &self - .target - .expect("include_target_in_component_name used in a targetless tarball"), - ); - } - let distdir = crate::dist::distdir(self.builder); - cmd.arg("generate") + build_cli(&self, &mut cmd); + cmd.arg("--rel-manifest-dir=rustlib") + .arg("--legacy-manifest-dirs=rustlib,cargo") .arg(format!("--product-name={}", self.product_name)) - .arg("--rel-manifest-dir=rustlib") .arg(format!("--success-message={} installed.", self.component)) - .arg("--image-dir") - .arg(self.image_dir) + .arg(format!("--package-name={}", package_name)) .arg("--work-dir") .arg(self.work_dir) .arg("--output-dir") - .arg(&distdir) - .arg("--non-installed-overlay") - .arg(self.overlay_dir) - .arg(format!("--package-name={}", package_name)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg(format!("--component-name={}", component_name)); + .arg(&distdir); self.builder.run(&mut cmd); - t!(std::fs::remove_dir_all(&self.temp_dir)); + if self.delete_temp_dir { + t!(std::fs::remove_dir_all(&self.temp_dir)); + } distdir.join(format!("{}.tar.gz", package_name)) } From f18335edb2917a310b69a522e6f3fde30af3d419 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 7 Dec 2020 11:49:22 +0100 Subject: [PATCH 20/23] bootstrap: convert rustc-src to use Tarball --- src/bootstrap/dist.rs | 34 ++----------------- src/bootstrap/tarball.rs | 73 +++++++++++++++++++++++----------------- 2 files changed, 45 insertions(+), 62 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index a68f139881744..258483bf1345b 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -37,10 +37,6 @@ pub fn tmpdir(builder: &Builder<'_>) -> PathBuf { builder.out.join("tmp/dist") } -fn rust_installer(builder: &Builder<'_>) -> Command { - builder.tool_cmd(Tool::RustInstaller) -} - fn missing_tool(tool_name: &str, skip: bool) { if skip { println!("Unable to build {}, skipping dist", tool_name) @@ -867,11 +863,8 @@ impl Step for PlainSourceTarball { /// Creates the plain source tarball fn run(self, builder: &Builder<'_>) -> PathBuf { - // Make sure that the root folder of tarball has the correct name - let plain_name = format!("{}-src", pkgname(builder, "rustc")); - let plain_dst_src = tmpdir(builder).join(&plain_name); - let _ = fs::remove_dir_all(&plain_dst_src); - t!(fs::create_dir_all(&plain_dst_src)); + let tarball = Tarball::new(builder, "rustc", "src"); + let plain_dst_src = tarball.image_dir(); // This is the set of root paths which will become part of the source package let src_files = [ @@ -914,28 +907,7 @@ impl Step for PlainSourceTarball { builder.run(&mut cmd); } - // Create plain source tarball - let plain_name = format!("rustc-{}-src", builder.rust_package_vers()); - let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name)); - tarball.set_extension(""); // strip .gz - tarball.set_extension(""); // strip .tar - if let Some(dir) = tarball.parent() { - builder.create_dir(&dir); - } - builder.info("running installer"); - let mut cmd = rust_installer(builder); - cmd.arg("tarball") - .arg("--input") - .arg(&plain_name) - .arg("--output") - .arg(&tarball) - .arg("--work-dir=.") - .current_dir(tmpdir(builder)); - - builder.info("Create plain source tarball"); - let _time = timeit(builder); - builder.run(&mut cmd); - distdir(builder).join(&format!("{}.tar.gz", plain_name)) + tarball.bare() } } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index b4146450596d5..5d73a655427b2 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -94,7 +94,6 @@ pub(crate) struct Tarball<'a> { temp_dir: PathBuf, image_dir: PathBuf, overlay_dir: PathBuf, - work_dir: PathBuf, include_target_in_component_name: bool, is_preview: bool, @@ -113,12 +112,14 @@ impl<'a> Tarball<'a> { fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option) -> Self { let pkgname = crate::dist::pkgname(builder, component); - let temp_dir = builder.out.join("tmp").join("tarball").join(component); + let mut temp_dir = builder.out.join("tmp").join("tarball"); + if let Some(target) = &target { + temp_dir = temp_dir.join(target); + } let _ = std::fs::remove_dir_all(&temp_dir); let image_dir = temp_dir.join("image"); let overlay_dir = temp_dir.join("overlay"); - let work_dir = temp_dir.join("work"); Self { builder, @@ -132,7 +133,6 @@ impl<'a> Tarball<'a> { temp_dir, image_dir, overlay_dir, - work_dir, include_target_in_component_name: false, is_preview: false, @@ -200,7 +200,7 @@ impl<'a> Tarball<'a> { pub(crate) fn persist_work_dir(&mut self) -> PathBuf { self.delete_temp_dir = false; - self.work_dir.clone() + self.temp_dir.clone() } pub(crate) fn generate(self) -> PathBuf { @@ -222,9 +222,8 @@ impl<'a> Tarball<'a> { cmd.arg("generate") .arg("--image-dir") .arg(&this.image_dir) - .arg("--non-installed-overlay") - .arg(&this.overlay_dir) .arg(format!("--component-name={}", &component_name)); + this.non_bare_args(cmd); }) } @@ -236,14 +235,41 @@ impl<'a> Tarball<'a> { } self.run(|this, cmd| { - cmd.arg("combine") - .arg("--input-tarballs") - .arg(input_tarballs) - .arg("--non-installed-overlay") - .arg(&this.overlay_dir); + cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs); + this.non_bare_args(cmd); }); } + pub(crate) fn bare(self) -> PathBuf { + self.run(|this, cmd| { + cmd.arg("tarball") + .arg("--input") + .arg(&this.image_dir) + .arg("--output") + .arg(crate::dist::distdir(this.builder).join(this.package_name())); + }) + } + + fn package_name(&self) -> String { + if let Some(target) = &self.target { + format!("{}-{}", self.pkgname, target) + } else { + self.pkgname.clone() + } + } + + fn non_bare_args(&self, cmd: &mut Command) { + cmd.arg("--rel-manifest-dir=rustlib") + .arg("--legacy-manifest-dirs=rustlib,cargo") + .arg(format!("--product-name={}", self.product_name)) + .arg(format!("--success-message={} installed.", self.component)) + .arg(format!("--package-name={}", self.package_name())) + .arg("--non-installed-overlay") + .arg(&self.overlay_dir) + .arg("--output-dir") + .arg(crate::dist::distdir(self.builder)); + } + fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> PathBuf { t!(std::fs::create_dir_all(&self.overlay_dir)); self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder)); @@ -256,32 +282,17 @@ impl<'a> Tarball<'a> { let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); - let package_name = if let Some(target) = &self.target { - self.builder.info(&format!("Dist {} ({})", self.component, target)); - format!("{}-{}", self.pkgname, target) - } else { - self.builder.info(&format!("Dist {}", self.component)); - self.pkgname.clone() - }; - + let package_name = self.package_name(); + self.builder.info(&format!("Dist {}", package_name)); let _time = crate::util::timeit(self.builder); - let distdir = crate::dist::distdir(self.builder); build_cli(&self, &mut cmd); - cmd.arg("--rel-manifest-dir=rustlib") - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg(format!("--product-name={}", self.product_name)) - .arg(format!("--success-message={} installed.", self.component)) - .arg(format!("--package-name={}", package_name)) - .arg("--work-dir") - .arg(self.work_dir) - .arg("--output-dir") - .arg(&distdir); + cmd.arg("--work-dir").arg(&self.temp_dir); self.builder.run(&mut cmd); if self.delete_temp_dir { t!(std::fs::remove_dir_all(&self.temp_dir)); } - distdir.join(format!("{}.tar.gz", package_name)) + crate::dist::distdir(self.builder).join(format!("{}.tar.gz", package_name)) } } From ae12a0c200da9a97d80d074e20bf4180fe4671b5 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 7 Dec 2020 17:23:26 +0100 Subject: [PATCH 21/23] bootstrap: avoid producing the rust tarball during dry runs --- src/bootstrap/dist.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 258483bf1345b..dddb9527ea1dc 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1288,6 +1288,11 @@ impl Step for Extended { let etc = builder.src.join("src/etc/installer"); + // Avoid producing tarballs during a dry run. + if builder.config.dry_run { + return; + } + // When rust-std package split from rustc, we needed to ensure that during // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering // the std files during uninstall. To do this ensure that rustc comes From 2c081769b09a876ecd3e9854f8cffc022ddc6b13 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 21 Dec 2020 18:04:58 +0100 Subject: [PATCH 22/23] bootstrap: use the normal compiler to build std --- src/bootstrap/dist.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index dddb9527ea1dc..6d9ee60f95276 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1283,8 +1283,7 @@ impl Step for Extended { let analysis_installer = builder.ensure(Analysis { compiler, target }); let docs_installer = builder.ensure(Docs { host: target }); - let std_installer = - builder.ensure(Std { compiler: builder.compiler(stage, target), target }); + let std_installer = builder.ensure(Std { compiler, target }); let etc = builder.src.join("src/etc/installer"); From 8736731f70be86b169a0dcc6a1900ccc05d2db85 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 23 Dec 2020 19:40:45 +0100 Subject: [PATCH 23/23] bootstrap: convert reproducible-artifacts to use Tarball --- src/bootstrap/dist.rs | 45 ++++--------------------------------------- 1 file changed, 4 insertions(+), 41 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 6d9ee60f95276..0a79d09b27fed 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2034,47 +2034,10 @@ impl Step for ReproducibleArtifacts { } fn run(self, builder: &Builder<'_>) -> Self::Output { - let name = pkgname(builder, "reproducible-artifacts"); - let tmp = tmpdir(builder); + let path = builder.config.rust_profile_use.as_ref()?; - // Prepare the image. - let image = tmp.join("reproducible-artifacts-image"); - let _ = fs::remove_dir_all(&image); - - if let Some(path) = &builder.config.rust_profile_use { - builder.install(std::path::Path::new(path), &image, 0o644); - } else { - return None; - } - - // Prepare the overlay. - let overlay = tmp.join("reproducible-artifacts-overlay"); - let _ = fs::remove_dir_all(&overlay); - builder.create_dir(&overlay); - builder.create(&overlay.join("version"), &builder.rust_version()); - for file in &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"] { - builder.install(&builder.src.join(file), &overlay, 0o644); - } - - // Create the final tarball. - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=reproducible-artifacts installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, self.target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=reproducible-artifacts"); - - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, self.target.triple))) + let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple); + tarball.add_file(path, ".", 0o644); + Some(tarball.generate()) } }