Auto merge of #41508 - michaelwoerister:generic-path-remapping, r=alexcrichton

Implement a file-path remapping feature in support of debuginfo and reproducible builds

This PR adds the `-Zremap-path-prefix-from`/`-Zremap-path-prefix-to` commandline option pair and is a more general implementation of #41419. As opposed to the previous attempt, this implementation should enable reproducible builds regardless of the working directory of the compiler.

This implementation of the feature is more general in the sense that the re-mapping will affect *all* paths the compiler emits, including the ones in error messages.

r? @alexcrichton

Author: bors

src/doc/unstable-book/src/SUMMARY.md

@@ -2,6 +2,7 @@
 
 - [Compiler flags](compiler-flags.md)
     - [linker_flavor](compiler-flags/linker-flavor.md)
+    - [remap_path_prefix](compiler-flags/remap-path-prefix.md)
 - [Language features](language-features.md)
     - [abi_msp430_interrupt](language-features/abi-msp430-interrupt.md)
     - [abi_ptx](language-features/abi-ptx.md)

src/doc/unstable-book/src/compiler-flags/remap-path-prefix.md

@@ -0,0 +1,37 @@
+# `remap-path-prefix`
+
+The tracking issue for this feature is: [#41555](https://github.com/rust-lang/rust/issues/41555)
+
+------------------------
+
+The `-Z remap-path-prefix-from`, `-Z remap-path-prefix-to` commandline option
+pair allows to replace prefixes of any file paths the compiler emits in various
+places. This is useful for bringing debuginfo paths into a well-known form and
+for achieving reproducible builds independent of the directory the compiler was
+executed in. All paths emitted by the compiler are affected, including those in
+error messages.
+
+In order to map all paths starting with `/home/foo/my-project/src` to
+`/sources/my-project`, one would invoke the compiler as follows:
+
+```text
+rustc -Zremap-path-prefix-from="/home/foo/my-project/src" -Zremap-path-prefix-to="/sources/my-project"
+```
+
+Debuginfo for code from the file `/home/foo/my-project/src/foo/mod.rs`,
+for example, would then point debuggers to `/sources/my-project/foo/mod.rs`
+instead of the original file.
+
+The options can be specified multiple times when multiple prefixes should be
+mapped:
+
+```text
+rustc -Zremap-path-prefix-from="/home/foo/my-project/src" \
+      -Zremap-path-prefix-to="/sources/my-project" \
+      -Zremap-path-prefix-from="/home/foo/my-project/build-dir" \
+      -Zremap-path-prefix-to="/stable-build-dir"
+```
+
+When the options are given multiple times, the nth `-from` will be matched up
+with the nth `-to` and they can appear anywhere on the commandline. Mappings
+specified later on the line will take precedence over earlier ones.

src/grammar/verify.rs

@@ -296,7 +296,7 @@ fn main() {
                                          syntax::errors::registry::Registry::new(&[]),
                                          Rc::new(DummyCrateStore));
     let filemap = session.parse_sess.codemap()
-                         .new_filemap("<n/a>".to_string(), None, code);
+                         .new_filemap("<n/a>".to_string(), code);
     let mut lexer = lexer::StringReader::new(session.diagnostic(), filemap);
     let cm = session.codemap();
 

src/librustc/session/config.rs

@@ -25,6 +25,7 @@ use lint;
 use middle::cstore;
 
 use syntax::ast::{self, IntTy, UintTy};
+use syntax::codemap::FilePathMapping;
 use syntax::parse::token;
 use syntax::parse;
 use syntax::symbol::Symbol;
@@ -492,6 +493,14 @@ impl Options {
         self.incremental.is_none() ||
         self.cg.codegen_units == 1
     }
+
+    pub fn file_path_mapping(&self) -> FilePathMapping {
+        FilePathMapping::new(
+            self.debugging_opts.remap_path_prefix_from.iter().zip(
+                self.debugging_opts.remap_path_prefix_to.iter()
+            ).map(|(src, dst)| (src.clone(), dst.clone())).collect()
+        )
+    }
 }
 
 // The type of entry function, so
@@ -1012,6 +1021,10 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         "Set the optimization fuel quota for a crate."),
     print_fuel: Option<String> = (None, parse_opt_string, [TRACKED],
         "Make Rustc print the total optimization fuel used by a crate."),
+    remap_path_prefix_from: Vec<String> = (vec![], parse_string_push, [TRACKED],
+        "add a source pattern to the file path remapping config"),
+    remap_path_prefix_to: Vec<String> = (vec![], parse_string_push, [TRACKED],
+        "add a mapping target to the file path remapping config"),
 }
 
 pub fn default_lib_output() -> CrateType {
@@ -1319,7 +1332,7 @@ pub fn rustc_optgroups() -> Vec<RustcOptGroup> {
 // Convert strings provided as --cfg [cfgspec] into a crate_cfg
 pub fn parse_cfgspecs(cfgspecs: Vec<String> ) -> ast::CrateConfig {
     cfgspecs.into_iter().map(|s| {
-        let sess = parse::ParseSess::new();
+        let sess = parse::ParseSess::new(FilePathMapping::empty());
         let mut parser =
             parse::new_parser_from_source_str(&sess, "cfgspec".to_string(), s.to_string());
 
@@ -1430,6 +1443,23 @@ pub fn build_session_options_and_crate_config(matches: &getopts::Matches)
         output_types.insert(OutputType::Exe, None);
     }
 
+    let remap_path_prefix_sources = debugging_opts.remap_path_prefix_from.len();
+    let remap_path_prefix_targets = debugging_opts.remap_path_prefix_from.len();
+
+    if remap_path_prefix_targets < remap_path_prefix_sources {
+        for source in &debugging_opts.remap_path_prefix_from[remap_path_prefix_targets..] {
+            early_error(error_format,
+                &format!("option `-Zremap-path-prefix-from='{}'` does not have \
+                         a corresponding `-Zremap-path-prefix-to`", source))
+        }
+    } else if remap_path_prefix_targets > remap_path_prefix_sources {
+        for target in &debugging_opts.remap_path_prefix_to[remap_path_prefix_sources..] {
+            early_error(error_format,
+                &format!("option `-Zremap-path-prefix-to='{}'` does not have \
+                          a corresponding `-Zremap-path-prefix-from`", target))
+        }
+    }
+
     let mut cg = build_codegen_options(matches, error_format);
 
     // Issue #30063: if user requests llvm-related output to one

src/librustc/session/mod.rs

@@ -74,8 +74,10 @@ pub struct Session {
     // The name of the root source file of the crate, in the local file system.
     // The path is always expected to be absolute. `None` means that there is no
     // source file.
-    pub local_crate_source_file: Option<PathBuf>,
-    pub working_dir: PathBuf,
+    pub local_crate_source_file: Option<String>,
+    // The directory the compiler has been executed in plus a flag indicating
+    // if the value stored here has been affected by path remapping.
+    pub working_dir: (String, bool),
     pub lint_store: RefCell<lint::LintStore>,
     pub lints: RefCell<lint::LintTable>,
     /// Set of (LintId, span, message) tuples tracking lint (sub)diagnostics
@@ -553,12 +555,14 @@ pub fn build_session(sopts: config::Options,
                      registry: errors::registry::Registry,
                      cstore: Rc<CrateStore>)
                      -> Session {
+    let file_path_mapping = sopts.file_path_mapping();
+
     build_session_with_codemap(sopts,
                                dep_graph,
                                local_crate_source_file,
                                registry,
                                cstore,
-                               Rc::new(codemap::CodeMap::new()),
+                               Rc::new(codemap::CodeMap::new(file_path_mapping)),
                                None)
 }
 
@@ -622,7 +626,7 @@ pub fn build_session_(sopts: config::Options,
         Ok(t) => t,
         Err(e) => {
             panic!(span_diagnostic.fatal(&format!("Error loading host specification: {}", e)));
-    }
+        }
     };
     let target_cfg = config::build_target_config(&sopts, &span_diagnostic);
     let p_s = parse::ParseSess::with_span_handler(span_diagnostic, codemap);
@@ -631,21 +635,22 @@ pub fn build_session_(sopts: config::Options,
         None => Some(filesearch::get_or_default_sysroot())
     };
 
+    let file_path_mapping = sopts.file_path_mapping();
+
     // Make the path absolute, if necessary
-    let local_crate_source_file = local_crate_source_file.map(|path|
-        if path.is_absolute() {
-            path.clone()
-        } else {
-            env::current_dir().unwrap().join(&path)
-        }
-    );
+    let local_crate_source_file = local_crate_source_file.map(|path| {
+        file_path_mapping.map_prefix(path.to_string_lossy().into_owned()).0
+    });
 
     let optimization_fuel_crate = sopts.debugging_opts.fuel.as_ref().map(|i| i.0.clone());
     let optimization_fuel_limit = Cell::new(sopts.debugging_opts.fuel.as_ref()
         .map(|i| i.1).unwrap_or(0));
     let print_fuel_crate = sopts.debugging_opts.print_fuel.clone();
     let print_fuel = Cell::new(0);
 
+    let working_dir = env::current_dir().unwrap().to_string_lossy().into_owned();
+    let working_dir = file_path_mapping.map_prefix(working_dir);
+
     let sess = Session {
         dep_graph: dep_graph.clone(),
         target: target_cfg,
@@ -660,7 +665,7 @@ pub fn build_session_(sopts: config::Options,
         derive_registrar_fn: Cell::new(None),
         default_sysroot: default_sysroot,
         local_crate_source_file: local_crate_source_file,
-        working_dir: env::current_dir().unwrap(),
+        working_dir: working_dir,
         lint_store: RefCell::new(lint::LintStore::new()),
         lints: RefCell::new(lint::LintTable::new()),
         one_time_diagnostics: RefCell::new(FxHashSet()),

src/librustc_driver/lib.rs

@@ -206,7 +206,7 @@ pub fn run_compiler<'a>(args: &[String],
     let cstore = Rc::new(CStore::new(&dep_graph));
 
     let loader = file_loader.unwrap_or(box RealFileLoader);
-    let codemap = Rc::new(CodeMap::with_file_loader(loader));
+    let codemap = Rc::new(CodeMap::with_file_loader(loader, sopts.file_path_mapping()));
     let mut sess = session::build_session_with_codemap(
         sopts, &dep_graph, input_file_path, descriptions, cstore.clone(), codemap, emitter_dest,
     );

src/librustc_driver/test.rs

@@ -31,7 +31,7 @@ use rustc::session::{self, config};
 use std::rc::Rc;
 use syntax::ast;
 use syntax::abi::Abi;
-use syntax::codemap::CodeMap;
+use syntax::codemap::{CodeMap, FilePathMapping};
 use errors;
 use errors::emitter::Emitter;
 use errors::{Level, DiagnosticBuilder};
@@ -108,7 +108,7 @@ fn test_env<F>(source_string: &str,
                                        &dep_graph,
                                        None,
                                        diagnostic_handler,
-                                       Rc::new(CodeMap::new()),
+                                       Rc::new(CodeMap::new(FilePathMapping::empty())),
                                        cstore.clone());
     rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
     let input = config::Input::Str {

src/librustc_metadata/cstore_impl.rs

@@ -397,7 +397,7 @@ impl CrateStore for cstore::CStore {
         let (name, def) = data.get_macro(id.index);
         let source_name = format!("<{} macros>", name);
 
-        let filemap = sess.parse_sess.codemap().new_filemap(source_name, None, def.body);
+        let filemap = sess.parse_sess.codemap().new_filemap(source_name, def.body);
         let local_span = Span { lo: filemap.start_pos, hi: filemap.end_pos, ctxt: NO_EXPANSION };
         let body = filemap_to_stream(&sess.parse_sess, filemap);
 

src/librustc_metadata/decoder.rs

@@ -1138,7 +1138,7 @@ impl<'a, 'tcx> CrateMetadata {
                         // We can't reuse an existing FileMap, so allocate a new one
                         // containing the information we need.
                         let syntax_pos::FileMap { name,
-                                                  abs_path,
+                                                  name_was_remapped,
                                                   start_pos,
                                                   end_pos,
                                                   lines,
@@ -1162,7 +1162,7 @@ impl<'a, 'tcx> CrateMetadata {
                         }
 
                         let local_version = local_codemap.new_imported_filemap(name,
-                                                                               abs_path,
+                                                                               name_was_remapped,
                                                                                source_length,
                                                                                lines,
                                                                                multibyte_chars);

src/librustc_metadata/encoder.rs

@@ -30,6 +30,7 @@ use std::hash::Hash;
 use std::intrinsics;
 use std::io::prelude::*;
 use std::io::Cursor;
+use std::path::Path;
 use std::rc::Rc;
 use std::u32;
 use syntax::ast::{self, CRATE_NODE_ID};
@@ -1270,13 +1271,40 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
     fn encode_codemap(&mut self) -> LazySeq<syntax_pos::FileMap> {
         let codemap = self.tcx.sess.codemap();
         let all_filemaps = codemap.files.borrow();
-        self.lazy_seq_ref(all_filemaps.iter()
+        let adapted = all_filemaps.iter()
             .filter(|filemap| {
                 // No need to re-export imported filemaps, as any downstream
                 // crate will import them from their original source.
                 !filemap.is_imported()
             })
-            .map(|filemap| &**filemap))
+            .map(|filemap| {
+                // When exporting FileMaps, we expand all paths to absolute
+                // paths because any relative paths are potentially relative to
+                // a wrong directory.
+                // However, if a path has been modified via
+                // `-Zremap-path-prefix` we assume the user has already set
+                // things up the way they want and don't touch the path values
+                // anymore.
+                let name = Path::new(&filemap.name);
+                let (ref working_dir, working_dir_was_remapped) = self.tcx.sess.working_dir;
+                if filemap.name_was_remapped ||
+                   (name.is_relative() && working_dir_was_remapped) {
+                    // This path of this FileMap has been modified by
+                    // path-remapping, so we use it verbatim (and avoid cloning
+                    // the whole map in the process).
+                    filemap.clone()
+                } else {
+                    let mut adapted = (**filemap).clone();
+                    let abs_path = Path::new(working_dir).join(name)
+                                                         .to_string_lossy()
+                                                         .into_owned();
+                    adapted.name = abs_path;
+                    Rc::new(adapted)
+                }
+            })
+            .collect::<Vec<_>>();
+
+        self.lazy_seq_ref(adapted.iter().map(|fm| &**fm))
     }
 
     fn encode_def_path_table(&mut self) -> Lazy<DefPathTable> {

src/librustc_save_analysis/dump_visitor.rs

@@ -37,6 +37,7 @@ use rustc::ty::{self, TyCtxt, AssociatedItemContainer};
 use std::collections::HashSet;
 use std::collections::hash_map::DefaultHasher;
 use std::hash::*;
+use std::path::Path;
 
 use syntax::ast::{self, NodeId, PatKind, Attribute, CRATE_NODE_ID};
 use syntax::parse::token;
@@ -128,6 +129,7 @@ impl<'l, 'tcx: 'l, 'll, D: Dump + 'll> DumpVisitor<'l, 'tcx, 'll, D> {
     pub fn dump_crate_info(&mut self, name: &str, krate: &ast::Crate) {
         let source_file = self.tcx.sess.local_crate_source_file.as_ref();
         let crate_root = source_file.map(|source_file| {
+            let source_file = Path::new(source_file);
             match source_file.file_name() {
                 Some(_) => source_file.parent().unwrap().display().to_string(),
                 None => source_file.display().to_string(),

src/librustc_trans/debuginfo/create_scope_map.rs

@@ -8,12 +8,12 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use super::FunctionDebugContext;
+use super::{FunctionDebugContext, FunctionDebugContextData};
 use super::metadata::file_metadata;
 use super::utils::{DIB, span_start};
 
 use llvm;
-use llvm::debuginfo::{DIScope, DISubprogram};
+use llvm::debuginfo::DIScope;
 use common::CrateContext;
 use rustc::mir::{Mir, VisibilityScope};
 
@@ -53,8 +53,8 @@ pub fn create_mir_scopes(ccx: &CrateContext, mir: &Mir, debug_context: &Function
     };
     let mut scopes = IndexVec::from_elem(null_scope, &mir.visibility_scopes);
 
-    let fn_metadata = match *debug_context {
-        FunctionDebugContext::RegularContext(ref data) => data.fn_metadata,
+    let debug_context = match *debug_context {
+        FunctionDebugContext::RegularContext(ref data) => data,
         FunctionDebugContext::DebugInfoDisabled |
         FunctionDebugContext::FunctionWithoutDebugInfo => {
             return scopes;
@@ -71,7 +71,7 @@ pub fn create_mir_scopes(ccx: &CrateContext, mir: &Mir, debug_context: &Function
     // Instantiate all scopes.
     for idx in 0..mir.visibility_scopes.len() {
         let scope = VisibilityScope::new(idx);
-        make_mir_scope(ccx, &mir, &has_variables, fn_metadata, scope, &mut scopes);
+        make_mir_scope(ccx, &mir, &has_variables, debug_context, scope, &mut scopes);
     }
 
     scopes
@@ -80,7 +80,7 @@ pub fn create_mir_scopes(ccx: &CrateContext, mir: &Mir, debug_context: &Function
 fn make_mir_scope(ccx: &CrateContext,
                   mir: &Mir,
                   has_variables: &BitVector,
-                  fn_metadata: DISubprogram,
+                  debug_context: &FunctionDebugContextData,
                   scope: VisibilityScope,
                   scopes: &mut IndexVec<VisibilityScope, MirDebugScope>) {
     if scopes[scope].is_valid() {
@@ -89,13 +89,13 @@ fn make_mir_scope(ccx: &CrateContext,
 
     let scope_data = &mir.visibility_scopes[scope];
     let parent_scope = if let Some(parent) = scope_data.parent_scope {
-        make_mir_scope(ccx, mir, has_variables, fn_metadata, parent, scopes);
+        make_mir_scope(ccx, mir, has_variables, debug_context, parent, scopes);
         scopes[parent]
     } else {
         // The root is the function itself.
         let loc = span_start(ccx, mir.span);
         scopes[scope] = MirDebugScope {
-            scope_metadata: fn_metadata,
+            scope_metadata: debug_context.fn_metadata,
             file_start_pos: loc.file.start_pos,
             file_end_pos: loc.file.end_pos,
         };
@@ -109,14 +109,17 @@ fn make_mir_scope(ccx: &CrateContext,
         // However, we don't skip creating a nested scope if
         // our parent is the root, because we might want to
         // put arguments in the root and not have shadowing.
-        if parent_scope.scope_metadata != fn_metadata {
+        if parent_scope.scope_metadata != debug_context.fn_metadata {
             scopes[scope] = parent_scope;
             return;
         }
     }
 
     let loc = span_start(ccx, scope_data.span);
-    let file_metadata = file_metadata(ccx, &loc.file.name, &loc.file.abs_path);
+    let file_metadata = file_metadata(ccx,
+                                      &loc.file.name,
+                                      debug_context.defining_crate);
+
     let scope_metadata = unsafe {
         llvm::LLVMRustDIBuilderCreateLexicalBlock(
             DIB(ccx),