Auto merge of #14282 - Veykril:rustc_private-proc-macro, r=Veykril

fix: Load proc-macros for rustc_private crates

Fixes #13591

Verified that this makes things work in the clippy repo (like resolving `sym` things for example)

Author: bors

crates/project-model/src/build_scripts.rs

@@ -15,13 +15,13 @@ use std::{
 
 use cargo_metadata::{camino::Utf8Path, Message};
 use la_arena::ArenaMap;
-use paths::AbsPathBuf;
+use paths::{AbsPath, AbsPathBuf};
 use rustc_hash::FxHashMap;
 use semver::Version;
 use serde::Deserialize;
 
 use crate::{
-    cfg_flag::CfgFlag, CargoConfig, CargoFeatures, CargoWorkspace, InvocationLocation,
+    cfg_flag::CfgFlag, utf8_stdout, CargoConfig, CargoFeatures, CargoWorkspace, InvocationLocation,
     InvocationStrategy, Package,
 };
 
@@ -250,7 +250,7 @@ impl WorkspaceBuildScripts {
 
         if tracing::enabled!(tracing::Level::INFO) {
             for package in workspace.packages() {
-                let package_build_data = &mut outputs[package];
+                let package_build_data = &outputs[package];
                 if !package_build_data.is_unchanged() {
                     tracing::info!(
                         "{}: {:?}",
@@ -378,6 +378,80 @@ impl WorkspaceBuildScripts {
     pub(crate) fn get_output(&self, idx: Package) -> Option<&BuildScriptOutput> {
         self.outputs.get(idx)
     }
+
+    pub(crate) fn rustc_crates(
+        rustc: &CargoWorkspace,
+        current_dir: &AbsPath,
+        extra_env: &FxHashMap<String, String>,
+    ) -> Self {
+        let mut bs = WorkspaceBuildScripts::default();
+        for p in rustc.packages() {
+            bs.outputs.insert(p, BuildScriptOutput::default());
+        }
+        let res = (|| {
+            let target_libdir = (|| {
+                let mut cargo_config = Command::new(toolchain::cargo());
+                cargo_config.envs(extra_env);
+                cargo_config
+                    .current_dir(current_dir)
+                    .args(["rustc", "-Z", "unstable-options", "--print", "target-libdir"])
+                    .env("RUSTC_BOOTSTRAP", "1");
+                if let Ok(it) = utf8_stdout(cargo_config) {
+                    return Ok(it);
+                }
+                let mut cmd = Command::new(toolchain::rustc());
+                cmd.envs(extra_env);
+                cmd.args(["--print", "target-libdir"]);
+                utf8_stdout(cmd)
+            })()?;
+
+            let target_libdir = AbsPathBuf::try_from(PathBuf::from(target_libdir))
+                .map_err(|_| anyhow::format_err!("target-libdir was not an absolute path"))?;
+            tracing::info!("Loading rustc proc-macro paths from {}", target_libdir.display());
+
+            let proc_macro_dylibs: Vec<(String, AbsPathBuf)> = std::fs::read_dir(target_libdir)?
+                .filter_map(|entry| {
+                    let dir_entry = entry.ok()?;
+                    if dir_entry.file_type().ok()?.is_file() {
+                        let path = dir_entry.path();
+                        tracing::info!("p{:?}", path);
+                        let extension = path.extension()?;
+                        if extension == "dll" || extension == "so" {
+                            let name = path.file_stem()?.to_str()?.split_once('-')?.0.to_owned();
+                            let path = AbsPathBuf::try_from(path).ok()?;
+                            return Some((name, path));
+                        }
+                    }
+                    None
+                })
+                .collect();
+            for p in rustc.packages() {
+                if let Some((_, path)) =
+                    proc_macro_dylibs.iter().find(|(name, _)| *name == rustc[p].name)
+                {
+                    bs.outputs[p].proc_macro_dylib_path = Some(path.clone());
+                }
+            }
+
+            if tracing::enabled!(tracing::Level::INFO) {
+                for package in rustc.packages() {
+                    let package_build_data = &bs.outputs[package];
+                    if !package_build_data.is_unchanged() {
+                        tracing::info!(
+                            "{}: {:?}",
+                            rustc[package].manifest.parent().display(),
+                            package_build_data,
+                        );
+                    }
+                }
+            }
+            Ok(())
+        })();
+        if let Err::<_, anyhow::Error>(e) = res {
+            bs.error = Some(e.to_string());
+        }
+        bs
+    }
 }
 
 // FIXME: Find a better way to know if it is a dylib.

crates/project-model/src/sysroot.rs

@@ -88,23 +88,17 @@ impl Sysroot {
     }
 
     pub fn discover_with_src_override(
-        dir: &AbsPath,
+        current_dir: &AbsPath,
         extra_env: &FxHashMap<String, String>,
         src: AbsPathBuf,
     ) -> Result<Sysroot> {
-        tracing::debug!("discovering sysroot for {}", dir.display());
-        let sysroot_dir = discover_sysroot_dir(dir, extra_env)?;
+        tracing::debug!("discovering sysroot for {}", current_dir.display());
+        let sysroot_dir = discover_sysroot_dir(current_dir, extra_env)?;
         Ok(Sysroot::load(sysroot_dir, src))
     }
 
-    pub fn discover_rustc(
-        cargo_toml: &ManifestPath,
-        extra_env: &FxHashMap<String, String>,
-    ) -> Option<ManifestPath> {
-        tracing::debug!("discovering rustc source for {}", cargo_toml.display());
-        let current_dir = cargo_toml.parent();
-        let sysroot_dir = discover_sysroot_dir(current_dir, extra_env).ok()?;
-        get_rustc_src(&sysroot_dir)
+    pub fn discover_rustc(&self) -> Option<ManifestPath> {
+        get_rustc_src(&self.root)
     }
 
     pub fn with_sysroot_dir(sysroot_dir: AbsPathBuf) -> Result<Sysroot> {

crates/project-model/src/workspace.rs

@@ -70,7 +70,7 @@ pub enum ProjectWorkspace {
         cargo: CargoWorkspace,
         build_scripts: WorkspaceBuildScripts,
         sysroot: Option<Sysroot>,
-        rustc: Option<CargoWorkspace>,
+        rustc: Option<(CargoWorkspace, WorkspaceBuildScripts)>,
         /// Holds cfg flags for the current target. We get those by running
         /// `rustc --print cfg`.
         ///
@@ -116,7 +116,7 @@ impl fmt::Debug for ProjectWorkspace {
                 .field("sysroot", &sysroot.is_some())
                 .field(
                     "n_rustc_compiler_crates",
-                    &rustc.as_ref().map_or(0, |rc| rc.packages().len()),
+                    &rustc.as_ref().map_or(0, |(rc, _)| rc.packages().len()),
                 )
                 .field("n_rustc_cfg", &rustc_cfg.len())
                 .field("n_cfg_overrides", &cfg_overrides.len())
@@ -243,7 +243,7 @@ impl ProjectWorkspace {
                 let rustc_dir = match &config.rustc_source {
                     Some(RustcSource::Path(path)) => ManifestPath::try_from(path.clone()).ok(),
                     Some(RustcSource::Discover) => {
-                        Sysroot::discover_rustc(&cargo_toml, &config.extra_env)
+                        sysroot.as_ref().and_then(Sysroot::discover_rustc)
                     }
                     None => None,
                 };
@@ -257,7 +257,15 @@ impl ProjectWorkspace {
                             config,
                             progress,
                         ) {
-                            Ok(meta) => Some(CargoWorkspace::new(meta)),
+                            Ok(meta) => {
+                                let workspace = CargoWorkspace::new(meta);
+                                let buildscripts = WorkspaceBuildScripts::rustc_crates(
+                                    &workspace,
+                                    cargo_toml.parent(),
+                                    &config.extra_env,
+                                );
+                                Some((workspace, buildscripts))
+                            }
                             Err(e) => {
                                 tracing::error!(
                                     %e,
@@ -531,7 +539,7 @@ impl ProjectWorkspace {
                         PackageRoot { is_local, include, exclude }
                     })
                     .chain(mk_sysroot(sysroot.as_ref(), Some(cargo.workspace_root())))
-                    .chain(rustc.iter().flat_map(|rustc| {
+                    .chain(rustc.iter().flat_map(|(rustc, _)| {
                         rustc.packages().map(move |krate| PackageRoot {
                             is_local: false,
                             include: vec![rustc[krate].manifest.parent().to_path_buf()],
@@ -559,7 +567,7 @@ impl ProjectWorkspace {
                 sysroot_package_len + project.n_crates()
             }
             ProjectWorkspace::Cargo { cargo, sysroot, rustc, .. } => {
-                let rustc_package_len = rustc.as_ref().map_or(0, |it| it.packages().len());
+                let rustc_package_len = rustc.as_ref().map_or(0, |(it, _)| it.packages().len());
                 let sysroot_package_len = sysroot.as_ref().map_or(0, |it| it.crates().len());
                 cargo.packages().len() + sysroot_package_len + rustc_package_len
             }
@@ -778,7 +786,7 @@ fn project_json_to_crate_graph(
 fn cargo_to_crate_graph(
     load_proc_macro: &mut dyn FnMut(&str, &AbsPath) -> ProcMacroLoadResult,
     load: &mut dyn FnMut(&AbsPath) -> Option<FileId>,
-    rustc: &Option<CargoWorkspace>,
+    rustc: &Option<(CargoWorkspace, WorkspaceBuildScripts)>,
     cargo: &CargoWorkspace,
     sysroot: Option<&Sysroot>,
     rustc_cfg: Vec<CfgFlag>,
@@ -924,7 +932,7 @@ fn cargo_to_crate_graph(
     if has_private {
         // If the user provided a path to rustc sources, we add all the rustc_private crates
         // and create dependencies on them for the crates which opt-in to that
-        if let Some(rustc_workspace) = rustc {
+        if let Some((rustc_workspace, build_scripts)) = rustc {
             handle_rustc_crates(
                 &mut crate_graph,
                 &mut pkg_to_lib_crate,