Auto merge of rust-lang#81746 - bjorn3:cg_clif_rustup_component, r=<try>

Distribute cg_clif as rustup component on the nightly channel

This makes it possible to use cg_clif using:

```bash
$ rustup component add rustc-codegen-cranelift-preview --toolchain nightly
$ RUSTFLAGS="-Zcodegen-backend=cranelift" cargo +nightly build
```

Marking as draft because I haven't verified that it correctly produces the component. In addition it makes cg_clif compile by default. This may want to be moved to the CI configuration instead.

cc rust-lang/compiler-team#405.
r? `@Mark-Simulacrum`

Author: bors

src/bootstrap/src/core/build_steps/dist.rs

@@ -27,7 +27,7 @@ use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
 use crate::core::config::TargetSelection;
 use crate::utils::cache::{Interned, INTERNER};
 use crate::utils::channel;
-use crate::utils::helpers::{exe, is_dylib, output, t, timeit};
+use crate::utils::helpers::{exe, is_dylib, output, t, target_supports_cranelift_backend, timeit};
 use crate::utils::tarball::{GeneratedTarball, OverlayKind, Tarball};
 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
 
@@ -443,19 +443,6 @@ impl Step for Rustc {
                 }
             }
 
-            // Copy over the codegen backends
-            let backends_src = builder.sysroot_codegen_backends(compiler);
-            let backends_rel = backends_src
-                .strip_prefix(&src)
-                .unwrap()
-                .strip_prefix(builder.sysroot_libdir_relative(compiler))
-                .unwrap();
-            // Don't use custom libdir here because ^lib/ will be resolved again with installer
-            let backends_dst = image.join("lib").join(&backends_rel);
-
-            t!(fs::create_dir_all(&backends_dst));
-            builder.cp_r(&backends_src, &backends_dst);
-
             // Copy libLLVM.so to the lib dir as well, if needed. While not
             // technically needed by rustc itself it's needed by lots of other
             // components like the llvm tools and LLD. LLD is included below and
@@ -1282,6 +1269,91 @@ impl Step for Miri {
     }
 }
 
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct CodegenBackend {
+    pub compiler: Compiler,
+    pub backend: Interned<String>,
+}
+
+impl Step for CodegenBackend {
+    type Output = Option<GeneratedTarball>;
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("compiler/rustc_codegen_cranelift")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        for &backend in &run.builder.config.rust_codegen_backends {
+            if backend == "llvm" {
+                continue; // Already built as part of rustc
+            }
+
+            run.builder.ensure(CodegenBackend {
+                compiler: run.builder.compiler(run.builder.top_stage, run.target),
+                backend,
+            });
+        }
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+        // This prevents rustc_codegen_cranelift from being built for "dist"
+        // or "install" on the stable/beta channels. It is not yet stable and
+        // should not be included.
+        if !builder.build.unstable_features() {
+            return None;
+        }
+
+        if self.backend == "cranelift" {
+            if !target_supports_cranelift_backend(self.compiler.host) {
+                builder.info("target not supported by rustc_codegen_cranelift. skipping");
+                return None;
+            }
+
+            if self.compiler.host.contains("windows") {
+                builder.info(
+                    "dist currently disabled for windows by rustc_codegen_cranelift. skipping",
+                );
+                return None;
+            }
+        }
+
+        let compiler = self.compiler;
+        let backend = self.backend;
+
+        let mut tarball =
+            Tarball::new(builder, &format!("rustc-codegen-{}", backend), &compiler.host.triple);
+        if backend == "cranelift" {
+            tarball.set_overlay(OverlayKind::RustcCodegenCranelift);
+        } else {
+            panic!("Unknown backend rustc_codegen_{}", backend);
+        }
+        tarball.is_preview(true);
+        tarball.add_legal_and_readme_to(format!("share/doc/rustc_codegen_{}", backend));
+
+        let src = builder.sysroot(compiler);
+        let backends_src = builder.sysroot_codegen_backends(compiler);
+        let backends_rel = backends_src
+            .strip_prefix(&src)
+            .unwrap()
+            .strip_prefix(builder.sysroot_libdir_relative(compiler))
+            .unwrap();
+        // Don't use custom libdir here because ^lib/ will be resolved again with installer
+        let backends_dst = PathBuf::from("lib").join(&backends_rel);
+
+        let backend_name = format!("rustc_codegen_{}", backend);
+        for backend in fs::read_dir(&backends_src).unwrap() {
+            let file_name = backend.unwrap().file_name();
+            if file_name.to_str().unwrap().contains(&backend_name) {
+                tarball.add_file(backends_src.join(file_name), &backends_dst, 0o644);
+            }
+        }
+
+        Some(tarball.generate())
+    }
+}
+
 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Rustfmt {
     pub compiler: Compiler,
@@ -1452,6 +1524,10 @@ impl Step for Extended {
         add_component!("clippy" => Clippy { compiler, target });
         add_component!("miri" => Miri { compiler, target });
         add_component!("analysis" => Analysis { compiler, target });
+        add_component!("rustc-codegen-cranelift" => CodegenBackend {
+            compiler: builder.compiler(stage, target),
+            backend: INTERNER.intern_str("cranelift"),
+        });
 
         let etc = builder.src.join("src/etc/installer");
 
@@ -1548,6 +1624,7 @@ impl Step for Extended {
                     prepare(tool);
                 }
             }
+            prepare("rustc-codegen-cranelift");
             // create an 'uninstall' package
             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
             pkgbuild("uninstall");
@@ -1587,6 +1664,10 @@ impl Step for Extended {
                     "rust-demangler-preview".to_string()
                 } else if name == "miri" {
                     "miri-preview".to_string()
+                } else if name == "rustc-codegen-cranelift" {
+                    // FIXME add installer support for cg_clif once it is ready to be distributed on
+                    // windows.
+                    unreachable!("cg_clif shouldn't be built for windows");
                 } else {
                     name.to_string()
                 };

src/bootstrap/src/core/build_steps/install.rs

@@ -13,6 +13,7 @@ use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
 use crate::core::config::{Config, TargetSelection};
 use crate::utils::helpers::t;
 use crate::utils::tarball::GeneratedTarball;
+use crate::INTERNER;
 use crate::{Compiler, Kind};
 
 #[cfg(target_os = "illumos")]
@@ -281,6 +282,19 @@ install!((self, builder, _config),
         });
         install_sh(builder, "rustc", self.compiler.stage, Some(self.target), &tarball);
     };
+    RustcCodegenCranelift, alias = "rustc-codegen-cranelift", Self::should_build(_config), only_hosts: true, {
+        if let Some(tarball) = builder.ensure(dist::CodegenBackend {
+            compiler: self.compiler,
+            backend: INTERNER.intern_str("cranelift"),
+        }) {
+            install_sh(builder, "rustc-codegen-cranelift", self.compiler.stage, Some(self.target), &tarball);
+        } else {
+            builder.info(
+                &format!("skipping Install CodegenBackend(\"cranelift\") stage{} ({})",
+                         self.compiler.stage, self.target),
+            );
+        }
+    };
 );
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]

src/bootstrap/src/core/build_steps/test.rs

@@ -28,7 +28,8 @@ use crate::core::config::TargetSelection;
 use crate::utils;
 use crate::utils::cache::{Interned, INTERNER};
 use crate::utils::helpers::{
-    self, add_link_lib_path, dylib_path, dylib_path_var, output, t, up_to_date,
+    self, add_link_lib_path, dylib_path, dylib_path_var, output, t,
+    target_supports_cranelift_backend, up_to_date,
 };
 use crate::utils::render_tests::{add_flags_and_try_run_tests, try_run_tests};
 use crate::{envify, CLang, DocTests, GitRepo, Mode};
@@ -2997,15 +2998,7 @@ impl Step for CodegenCranelift {
             return;
         }
 
-        let triple = run.target.triple;
-        let target_supported = if triple.contains("linux") {
-            triple.contains("x86_64") || triple.contains("aarch64") || triple.contains("s390x")
-        } else if triple.contains("darwin") || triple.contains("windows") {
-            triple.contains("x86_64")
-        } else {
-            false
-        };
-        if !target_supported {
+        if !target_supports_cranelift_backend(run.target) {
             builder.info("target not supported by rustc_codegen_cranelift. skipping");
             return;
         }

src/bootstrap/src/core/builder.rs

@@ -815,6 +815,7 @@ impl<'a> Builder<'a> {
                 dist::JsonDocs,
                 dist::Mingw,
                 dist::Rustc,
+                dist::CodegenBackend,
                 dist::Std,
                 dist::RustcDev,
                 dist::Analysis,

src/bootstrap/src/utils/helpers.rs

@@ -183,6 +183,16 @@ pub fn use_host_linker(target: TargetSelection) -> bool {
         || target.contains("switch"))
 }
 
+pub fn target_supports_cranelift_backend(target: TargetSelection) -> bool {
+    if target.contains("linux") {
+        target.contains("x86_64") || target.contains("aarch64") || target.contains("s390x")
+    } else if target.contains("darwin") || target.contains("windows") {
+        target.contains("x86_64")
+    } else {
+        false
+    }
+}
+
 pub fn is_valid_test_suite_arg<'a, P: AsRef<Path>>(
     path: &'a Path,
     suite_path: P,

src/bootstrap/src/utils/tarball.rs

@@ -19,6 +19,7 @@ pub(crate) enum OverlayKind {
     RustDemangler,
     RLS,
     RustAnalyzer,
+    RustcCodegenCranelift,
 }
 
 impl OverlayKind {
@@ -58,6 +59,11 @@ impl OverlayKind {
                 "src/tools/rust-analyzer/LICENSE-APACHE",
                 "src/tools/rust-analyzer/LICENSE-MIT",
             ],
+            OverlayKind::RustcCodegenCranelift => &[
+                "compiler/rustc_codegen_cranelift/Readme.md",
+                "compiler/rustc_codegen_cranelift/LICENSE-APACHE",
+                "compiler/rustc_codegen_cranelift/LICENSE-MIT",
+            ],
         }
     }
 
@@ -80,6 +86,7 @@ impl OverlayKind {
             OverlayKind::RustAnalyzer => builder
                 .rust_analyzer_info
                 .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")),
+            OverlayKind::RustcCodegenCranelift => builder.rust_version(),
         }
     }
 }

src/ci/run.sh

@@ -104,6 +104,8 @@ if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-assertions"
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
   fi
+
+  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift"
 else
   # We almost always want debug assertions enabled, but sometimes this takes too
   # long for too little benefit, so we just turn them off.
@@ -124,7 +126,6 @@ else
 
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
 
-  # Test the Cranelift backend in on CI, but don't ship it.
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift"
 
   # We enable this for non-dist builders, since those aren't trying to produce

src/tools/build-manifest/src/main.rs

@@ -191,7 +191,8 @@ static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"
 
 static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
 
-static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = &[PkgType::Miri, PkgType::JsonDocs];
+static NIGHTLY_ONLY_COMPONENTS: &[PkgType] =
+    &[PkgType::Miri, PkgType::JsonDocs, PkgType::RustcCodegenCranelift];
 
 macro_rules! t {
     ($e:expr) => {
@@ -335,7 +336,15 @@ impl Builder {
 
         // NOTE: this profile is effectively deprecated; do not add new components to it.
         let mut complete = default;
-        complete.extend([Rls, RustAnalyzer, RustSrc, LlvmTools, RustAnalysis, Miri]);
+        complete.extend([
+            Rls,
+            RustAnalyzer,
+            RustSrc,
+            LlvmTools,
+            RustAnalysis,
+            Miri,
+            RustcCodegenCranelift,
+        ]);
         profile("complete", &complete);
 
         // The compiler libraries are not stable for end users, and they're also huge, so we only
@@ -422,7 +431,8 @@ impl Builder {
                 | PkgType::Rustfmt
                 | PkgType::LlvmTools
                 | PkgType::RustAnalysis
-                | PkgType::JsonDocs => {
+                | PkgType::JsonDocs
+                | PkgType::RustcCodegenCranelift => {
                     extensions.push(host_component(pkg));
                 }
                 PkgType::RustcDev | PkgType::RustcDocs => {

src/tools/build-manifest/src/versions.rs

@@ -57,6 +57,7 @@ pkg_type! {
     LlvmTools = "llvm-tools"; preview = true,
     Miri = "miri"; preview = true,
     JsonDocs = "rust-docs-json"; preview = true,
+    RustcCodegenCranelift = "rustc-codegen-cranelift"; preview = true,
 }
 
 impl PkgType {
@@ -80,6 +81,7 @@ impl PkgType {
             PkgType::Rustfmt => false,
             PkgType::LlvmTools => false,
             PkgType::Miri => false,
+            PkgType::RustcCodegenCranelift => false,
 
             PkgType::Rust => true,
             PkgType::RustStd => true,
@@ -106,6 +108,8 @@ impl PkgType {
             ReproducibleArtifacts => HOSTS,
             RustcDocs => HOSTS,
             Cargo => HOSTS,
+            // FIXME should this use the exact list of targets for which we build cg_clif?
+            RustcCodegenCranelift => HOSTS,
             RustMingw => MINGW,
             RustStd => TARGETS,
             HtmlDocs => HOSTS,