Skip to content

Commit fffeaa7

Browse files
Dylan-DPCDylan-DPC
authored
Rollup merge of #77407 - pietroalbini:less-build-manifest, r=Mark-Simulacrum
Improve build-manifest to work with the improved promote-release This PR makes some changes to build-manifest to have it work better with the other improvements I'm making to [promote-release](https://github.com/rust-lang/promote-release). A new way to invoke the tool was added: `./x.py run src/tools/build-manifest`. The new invocation disables the generation of `.sha256` files and the generation of GPG signatures, as those steps are not tied to the Rust version we're building the manifest of: handling them in `promote-release` will improve the maintenability of our release process. Invocations through the old command (`./x.py dist hash-and-sign`) are referred inside the source code as "legacy". The new invocation also enables internal parallelism, disabled on legacy to avoid overloading our old server. Improvements were also made on how the checksums included in the manifest are generated: * The manifest is first generated with placeholder checksums, and then a function walks through the manifes and calculates only the needed hashes. Before this PR, all the hashes were calculated beforehand, including the hashes of unused files. * Calculating the hashes is now done in parallel with rayon, to better utilize all the available disk bandwidth. * The `sha2` crate is now used instead of the `sha256sum` CLI tool: this avoids the overhead of calling another process, but more importantly enables hardware acceleration whenever available (the `sha256sum` CLI tool doesn't support it at all). r? @Mark-Simulacrum This PR is best reviewed commit-by-commit.
2 parents fe087ec + 9352062 commit fffeaa7

File tree

8 files changed

+334
-130
lines changed

8 files changed

+334
-130
lines changed

‎Cargo.lock

Lines changed: 64 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,16 @@ dependencies = [
183183
"block-padding",
184184
"byte-tools",
185185
"byteorder",
186-
"generic-array",
186+
"generic-array 0.12.3",
187+
]
188+
189+
[[package]]
190+
name = "block-buffer"
191+
version = "0.9.0"
192+
source = "registry+https://github.com/rust-lang/crates.io-index"
193+
checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
194+
dependencies = [
195+
"generic-array 0.14.4",
187196
]
188197

189198
[[package]]
@@ -233,8 +242,11 @@ version = "0.1.0"
233242
dependencies = [
234243
"anyhow",
235244
"flate2",
245+
"hex 0.4.2",
246+
"rayon",
236247
"serde",
237248
"serde_json",
249+
"sha2",
238250
"tar",
239251
"toml",
240252
]
@@ -687,6 +699,12 @@ version = "0.8.0"
687699
source = "registry+https://github.com/rust-lang/crates.io-index"
688700
checksum = "9a21fa21941700a3cd8fcb4091f361a6a712fac632f85d9f487cc892045d55c6"
689701

702+
[[package]]
703+
name = "cpuid-bool"
704+
version = "0.1.2"
705+
source = "registry+https://github.com/rust-lang/crates.io-index"
706+
checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634"
707+
690708
[[package]]
691709
name = "crates-io"
692710
version = "0.31.1"
@@ -884,7 +902,16 @@ version = "0.8.1"
884902
source = "registry+https://github.com/rust-lang/crates.io-index"
885903
checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
886904
dependencies = [
887-
"generic-array",
905+
"generic-array 0.12.3",
906+
]
907+
908+
[[package]]
909+
name = "digest"
910+
version = "0.9.0"
911+
source = "registry+https://github.com/rust-lang/crates.io-index"
912+
checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
913+
dependencies = [
914+
"generic-array 0.14.4",
888915
]
889916

890917
[[package]]
@@ -1166,6 +1193,16 @@ dependencies = [
11661193
"typenum",
11671194
]
11681195

1196+
[[package]]
1197+
name = "generic-array"
1198+
version = "0.14.4"
1199+
source = "registry+https://github.com/rust-lang/crates.io-index"
1200+
checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817"
1201+
dependencies = [
1202+
"typenum",
1203+
"version_check",
1204+
]
1205+
11691206
[[package]]
11701207
name = "getopts"
11711208
version = "0.2.21"
@@ -1835,9 +1872,9 @@ version = "0.8.0"
18351872
source = "registry+https://github.com/rust-lang/crates.io-index"
18361873
checksum = "a18af3dcaf2b0219366cdb4e2af65a6101457b415c3d1a5c71dd9c2b7c77b9c8"
18371874
dependencies = [
1838-
"block-buffer",
1839-
"digest",
1840-
"opaque-debug",
1875+
"block-buffer 0.7.3",
1876+
"digest 0.8.1",
1877+
"opaque-debug 0.2.3",
18411878
]
18421879

18431880
[[package]]
@@ -2097,6 +2134,12 @@ version = "0.2.3"
20972134
source = "registry+https://github.com/rust-lang/crates.io-index"
20982135
checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
20992136

2137+
[[package]]
2138+
name = "opaque-debug"
2139+
version = "0.3.0"
2140+
source = "registry+https://github.com/rust-lang/crates.io-index"
2141+
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
2142+
21002143
[[package]]
21012144
name = "open"
21022145
version = "1.4.0"
@@ -4362,10 +4405,23 @@ version = "0.8.2"
43624405
source = "registry+https://github.com/rust-lang/crates.io-index"
43634406
checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df"
43644407
dependencies = [
4365-
"block-buffer",
4366-
"digest",
4408+
"block-buffer 0.7.3",
4409+
"digest 0.8.1",
43674410
"fake-simd",
4368-
"opaque-debug",
4411+
"opaque-debug 0.2.3",
4412+
]
4413+
4414+
[[package]]
4415+
name = "sha2"
4416+
version = "0.9.1"
4417+
source = "registry+https://github.com/rust-lang/crates.io-index"
4418+
checksum = "2933378ddfeda7ea26f48c555bdad8bb446bf8a3d17832dc83e380d444cfb8c1"
4419+
dependencies = [
4420+
"block-buffer 0.9.0",
4421+
"cfg-if",
4422+
"cpuid-bool",
4423+
"digest 0.9.0",
4424+
"opaque-debug 0.3.0",
43694425
]
43704426

43714427
[[package]]

‎src/bootstrap/builder.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -477,7 +477,7 @@ impl<'a> Builder<'a> {
477477
install::Src,
478478
install::Rustc
479479
),
480-
Kind::Run => describe!(run::ExpandYamlAnchors,),
480+
Kind::Run => describe!(run::ExpandYamlAnchors, run::BuildManifest,),
481481
}
482482
}
483483

‎src/bootstrap/dist.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ pub fn pkgname(builder: &Builder<'_>, component: &str) -> String {
4646
}
4747
}
4848

49-
fn distdir(builder: &Builder<'_>) -> PathBuf {
49+
pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf {
5050
builder.out.join("dist")
5151
}
5252

@@ -2371,6 +2371,7 @@ impl Step for HashSign {
23712371
cmd.arg(addr);
23722372
cmd.arg(&builder.config.channel);
23732373
cmd.arg(&builder.src);
2374+
cmd.env("BUILD_MANIFEST_LEGACY", "1");
23742375

23752376
builder.create_dir(&distdir(builder));
23762377

‎src/bootstrap/run.rs

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
use crate::builder::{Builder, RunConfig, ShouldRun, Step};
2+
use crate::dist::distdir;
23
use crate::tool::Tool;
4+
use build_helper::output;
35
use std::process::Command;
46

57
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
@@ -41,3 +43,43 @@ fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool {
4143
}
4244
true
4345
}
46+
47+
#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
48+
pub struct BuildManifest;
49+
50+
impl Step for BuildManifest {
51+
type Output = ();
52+
const ONLY_HOSTS: bool = true;
53+
54+
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
55+
run.path("src/tools/build-manifest")
56+
}
57+
58+
fn make_run(run: RunConfig<'_>) {
59+
run.builder.ensure(BuildManifest);
60+
}
61+
62+
fn run(self, builder: &Builder<'_>) {
63+
// This gets called by `promote-release`
64+
// (https://github.com/rust-lang/promote-release).
65+
let mut cmd = builder.tool_cmd(Tool::BuildManifest);
66+
let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| {
67+
panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n")
68+
});
69+
let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| {
70+
panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n")
71+
});
72+
73+
let today = output(Command::new("date").arg("+%Y-%m-%d"));
74+
75+
cmd.arg(sign);
76+
cmd.arg(distdir(builder));
77+
cmd.arg(today.trim());
78+
cmd.arg(addr);
79+
cmd.arg(&builder.config.channel);
80+
cmd.arg(&builder.src);
81+
82+
builder.create_dir(&distdir(builder));
83+
builder.run(&mut cmd);
84+
}
85+
}

‎src/tools/build-manifest/Cargo.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,6 @@ serde_json = "1.0"
1111
anyhow = "1.0.32"
1212
flate2 = "1.0.16"
1313
tar = "0.4.29"
14+
sha2 = "0.9.1"
15+
rayon = "1.3.1"
16+
hex = "0.4.2"

‎src/tools/build-manifest/README.md

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to
2020
`path/to/output` with:
2121

2222
```
23-
$ BUILD_MANIFEST_DISABLE_SIGNING=1 cargo +nightly run \
24-
path/to/dist path/to/output 1970-01-01 http://example.com \
23+
$ cargo +nightly run path/to/dist path/to/output 1970-01-01 http://example.com \
2524
CHANNEL path/to/rust/repo
2625
```
2726

0 commit comments

Comments
 (0)