Combine run-pass tests into a single crate

Author: Zoxc

config.toml.example

@@ -283,6 +283,9 @@
 # desired in distributions, for example.
 #rpath = true
 
+# Merge Rust test files into larger units for faster testing
+#combine-tests = false
+
 # Suppresses extraneous output from tests to ensure the output of the test
 # harness is relatively clean.
 #quiet-tests = false

src/Cargo.lock

@@ -119,6 +119,7 @@ pub struct Config {
     pub low_priority: bool,
     pub channel: String,
     pub quiet_tests: bool,
+    pub combine_tests: bool,
     pub test_miri: bool,
     pub save_toolstates: Option<PathBuf>,
 
@@ -288,6 +289,7 @@ struct Rust {
     debug: Option<bool>,
     dist_src: Option<bool>,
     quiet_tests: Option<bool>,
+    combine_tests: Option<bool>,
     test_miri: Option<bool>,
     save_toolstates: Option<String>,
     codegen_backends: Option<Vec<String>>,
@@ -478,6 +480,7 @@ impl Config {
             set(&mut config.backtrace, rust.backtrace);
             set(&mut config.channel, rust.channel.clone());
             set(&mut config.rust_dist_src, rust.dist_src);
+            set(&mut config.combine_tests, rust.combine_tests);
             set(&mut config.quiet_tests, rust.quiet_tests);
             set(&mut config.test_miri, rust.test_miri);
             set(&mut config.wasm_syscall, rust.wasm_syscall);

src/bootstrap/config.rs

@@ -47,6 +47,7 @@ def v(*args):
 o("test-miri", "rust.test-miri", "run miri's test suite")
 o("debuginfo-tests", "rust.debuginfo-tests", "build tests with debugger metadata")
 o("quiet-tests", "rust.quiet-tests", "enable quieter output when running tests")
+o("combined-tests", "rust.combine-tests", "merge tests together when possible")
 o("ccache", "llvm.ccache", "invoke gcc/clang via ccache to reuse object files between builds")
 o("sccache", None, "invoke gcc/clang via sccache to reuse object files between builds")
 o("local-rust", None, "use an installed rustc rather than downloading a snapshot")

src/bootstrap/configure.py

@@ -848,6 +848,10 @@ impl Step for Compiletest {
         cmd.arg("--run-lib-path").arg(builder.sysroot_libdir(compiler, target));
         cmd.arg("--rustc-path").arg(builder.rustc(compiler));
 
+        if builder.config.combine_tests && mode == "run-pass" {
+            cmd.arg("--combine");
+        }
+
         // Avoid depending on rustdoc when we don't need it.
         if mode == "rustdoc" || mode == "run-make" {
             cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler.host));

src/bootstrap/test.rs

@@ -1114,6 +1114,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         "generate comments into the assembly (may change behavior)"),
     no_verify: bool = (false, parse_bool, [TRACKED],
         "skip LLVM verification"),
+    submodules_crate_like: bool = (false, parse_bool, [TRACKED],
+        "pretend immediate submodules of the crate are crates themselves"),
     borrowck_stats: bool = (false, parse_bool, [UNTRACKED],
         "gather borrowck statistics"),
     no_landing_pads: bool = (false, parse_bool, [TRACKED],

src/librustc/session/config.rs

@@ -1095,7 +1095,8 @@ pub fn build_session_(
     };
     let target_cfg = config::build_target_config(&sopts, &span_diagnostic);
 
-    let p_s = parse::ParseSess::with_span_handler(span_diagnostic, codemap);
+    let mut p_s = parse::ParseSess::with_span_handler(span_diagnostic, codemap);
+    p_s.combine_test_mode = sopts.debugging_opts.submodules_crate_like;
     let default_sysroot = match sopts.maybe_sysroot {
         Some(_) => None,
         None => Some(filesearch::get_or_default_sysroot()),

src/librustc/session/mod.rs

@@ -521,7 +521,7 @@ pub fn phase_1_parse_input<'a>(control: &CompileController,
         profile::begin(sess);
     }
 
-    let krate = time(sess, "parsing", || {
+    let mut krate = time(sess, "parsing", || {
         match *input {
             Input::File(ref file) => {
                 parse::parse_crate_from_file(file, &sess.parse_sess)
@@ -549,6 +549,17 @@ pub fn phase_1_parse_input<'a>(control: &CompileController,
         syntax::show_span::run(sess.diagnostic(), s, &krate);
     }
 
+    if sess.opts.debugging_opts.submodules_crate_like {
+        krate = time(sess,
+            "Pretending submodules are crates",
+            || rustc_passes::submodules_crate_like::modify_crate(sess, krate));
+    }
+
+    if sess.opts.debugging_opts.ast_json_noexpand {
+        println!("\n\n\n\n\n\n\n\n");
+        println!("{}", json::as_json(&krate));
+    }
+
     if sess.opts.debugging_opts.hir_stats {
         hir_stats::print_ast_stats(&krate, "PRE EXPANSION AST STATS");
     }

src/librustc_driver/driver.rs

@@ -895,8 +895,8 @@ impl<'a> CompilerCalls<'a> for RustcDefaultCalls {
         }
 
         if sess.opts.debugging_opts.parse_only ||
-           sess.opts.debugging_opts.show_span.is_some() ||
-           sess.opts.debugging_opts.ast_json_noexpand {
+           sess.opts.debugging_opts.show_span.is_some()/* ||
+           sess.opts.debugging_opts.ast_json_noexpand*/ {
             control.after_parse.stop = Compilation::Stop;
         }
 

src/librustc_driver/lib.rs

@@ -38,6 +38,7 @@ use rustc::ty::maps::Providers;
 
 mod diagnostics;
 
+pub mod submodules_crate_like;
 pub mod ast_validation;
 pub mod rvalue_promotion;
 pub mod hir_stats;

src/librustc_passes/lib.rs

@@ -0,0 +1,214 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use rustc::session::Session;
+use syntax::ast::*;
+use syntax::ptr::P;
+use syntax::fold;
+use syntax::fold::Folder;
+use syntax::symbol::keywords;
+use syntax::codemap::dummy_spanned;
+use syntax_pos::DUMMY_SP;
+use syntax_pos::symbol::Symbol;
+
+struct ModFolder<'a> {
+    use_std: &'a Item,
+    root: Option<Ident>,
+    main: Symbol,
+}
+
+impl<'a> ModFolder<'a> {
+    fn fold_qpath(&mut self, qself: &mut Option<QSelf>, path: &mut Path) {
+        let old = path.segments.len();
+        *path = self.fold_path(path.clone());
+        let add = path.segments.len() - old;
+        qself.as_mut().map(|qself| {
+            qself.position += add;
+            qself.ty = self.fold_ty(qself.ty.clone());
+        });
+    }
+}
+
+impl<'a> fold::Folder for ModFolder<'a> {
+    fn fold_use_tree(&mut self, mut use_tree: UseTree) -> UseTree {
+        if let Some(root) = self.root {
+            let pos = {
+                let get = |i| {
+                    use_tree.prefix.segments.get(i).map(|p: &PathSegment| p.identifier.name)
+                };
+                if get(0) == Some(keywords::SelfValue.name()) ||
+                   get(0) == Some(keywords::Super.name()) {
+                    None
+                } else {
+                    let mut i = 0;
+                    if get(i) == Some(keywords::CrateRoot.name()) {
+                        i += 1;
+                    }
+                    if get(i) == Some(keywords::Crate.name()) {
+                        i += 1;
+                    }
+                    Some(i)
+                }
+            };
+            if let Some(pos) = pos {
+                use_tree.prefix.segments.insert(pos, PathSegment {
+                    identifier: root,
+                    span: use_tree.span,
+                    parameters: None,
+                });
+            }
+            use_tree
+        } else {
+            fold::noop_fold_use_tree(use_tree, self)
+        }
+    }
+
+    fn fold_path(&mut self, mut p: Path) -> Path {
+        if let Some(root) = self.root {
+            if let Some(first) = p.segments.first().cloned() {
+                if first.identifier.name == keywords::CrateRoot.name() {
+                    let idx = if p.segments.get(1).map(|p| p.identifier.name) ==
+                                    Some(keywords::Crate.name()) {
+                        2
+                    } else {
+                        1
+                    };
+                    p.segments.insert(idx, PathSegment {
+                        identifier: root,
+                        span: p.span,
+                        parameters: None,
+                    });
+                }
+            }
+            fold::noop_fold_path(p, self)
+        } else {
+            fold::noop_fold_path(p, self)
+        }
+    }
+
+    fn fold_ty(&mut self, mut t: P<Ty>) -> P<Ty> {
+        if match t.node  {
+            TyKind::Path(ref mut qself, ref mut path) => {
+                self.fold_qpath(qself, path);
+                true
+            }
+            _ => false,
+        } {
+            return t;
+        }
+        fold::noop_fold_ty(t, self)
+    }
+
+    fn fold_pat(&mut self, mut p: P<Pat>) -> P<Pat> {
+        if match p.node  {
+            PatKind::Path(ref mut qself, ref mut path) => {
+                self.fold_qpath(qself, path);
+                true
+            }
+            _ => false,
+        } {
+            return p;
+        }
+        fold::noop_fold_pat(p, self)
+    }
+
+    fn fold_expr(&mut self, mut e: P<Expr>) -> P<Expr> {
+        if match e.node  {
+            ExprKind::Path(ref mut qself, ref mut path) => {
+                self.fold_qpath(qself, path);
+                true
+            }
+            _ => false,
+        } {
+            return e;
+        }
+        e.map(|e| fold::noop_fold_expr(e, self))
+    }
+
+    fn fold_item_simple(&mut self, mut item: Item) -> Item {
+        if self.root.is_some() {
+            fold::noop_fold_item_simple(item, self)
+        } else {
+            let is_root = match item.node {
+                ItemKind::Mod(ref mut module) => {
+                    for mut item in &mut module.items {
+                        if item.ident.name == self.main {
+                            item.vis = dummy_spanned(VisibilityKind::Public);
+                        }
+                    }
+                    true
+                }
+                _ => false,
+            };
+
+            if is_root {
+                let mut folder = ModFolder {
+                    use_std: self.use_std,
+                    root: Some(item.ident),
+                    main: self.main,
+                };
+
+                let mut item = fold::noop_fold_item_simple(item, &mut folder);
+                match item.node {
+                    ItemKind::Mod(ref mut module) => {
+                        module.items.push(P(self.use_std.clone()));
+                    }
+                    _ => (),
+                }
+                item
+            } else {
+                fold::noop_fold_item_simple(item, self)
+            }
+        }
+    }
+
+    fn fold_mac(&mut self, _mac: Mac) -> Mac {
+        fold::noop_fold_mac(_mac, self)
+    }
+}
+
+pub fn modify_crate(_session: &Session, krate: Crate) -> Crate {
+    let std_i = Ident::from_str("std");
+    let use_std = Item {
+        ident: keywords::Invalid.ident(),
+        attrs: Vec::new(),
+        id: DUMMY_NODE_ID,
+        node: ItemKind::Use(P(UseTree {
+            span: DUMMY_SP,
+            kind: UseTreeKind::Simple(Some(std_i)),
+            prefix: Path {
+                segments: vec![PathSegment {
+                    identifier: std_i,
+                    span: DUMMY_SP,
+                    parameters: None,
+                }],
+                span: DUMMY_SP,
+            },
+        })),
+        vis: dummy_spanned(VisibilityKind::Inherited),
+        span: DUMMY_SP,
+        tokens: None,
+    };
+    let Crate {
+        module,
+        attrs,
+        span,
+    } = krate;
+    let module = ModFolder {
+        use_std: &use_std,
+        root: None,
+        main: Symbol::intern("main"),
+    }.fold_mod(module);
+    Crate {
+        module,
+        attrs,
+        span,
+    }
+}

src/librustc_passes/submodules_crate_like.rs

@@ -1187,12 +1187,17 @@ pub fn feature_err<'a>(sess: &'a ParseSess, feature: &str, span: Span, issue: Ga
 }
 
 fn leveled_feature_err<'a>(sess: &'a ParseSess, feature: &str, span: Span, issue: GateIssue,
-                           explain: &str, level: GateStrength) -> DiagnosticBuilder<'a> {
+                           explain: &str, mut level: GateStrength) -> DiagnosticBuilder<'a> {
     let diag = &sess.span_diagnostic;
 
     let issue = match issue {
         GateIssue::Language => find_lang_feature_issue(feature),
-        GateIssue::Library(lib) => lib,
+        GateIssue::Library(lib) => {
+            if sess.combine_test_mode {
+                level = GateStrength::Soft;
+            }
+            lib
+        },
     };
 
     let explanation = if let Some(n) = issue {

src/libsyntax/feature_gate.rs

@@ -53,6 +53,7 @@ pub struct ParseSess {
     // Spans where a `mod foo;` statement was included in a non-mod.rs file.
     // These are used to issue errors if the non_modrs_mods feature is not enabled.
     pub non_modrs_mods: RefCell<Vec<(ast::Ident, Span)>>,
+    pub combine_test_mode: bool,
     /// Used to determine and report recursive mod inclusions
     included_mod_stack: RefCell<Vec<PathBuf>>,
     code_map: Lrc<CodeMap>,
@@ -77,6 +78,7 @@ impl ParseSess {
             registered_diagnostics: Lock::new(ErrorMap::new()),
             included_mod_stack: RefCell::new(vec![]),
             code_map,
+            combine_test_mode: false,
             non_modrs_mods: RefCell::new(vec![]),
         }
     }

src/libsyntax/parse/mod.rs

@@ -38,7 +38,7 @@
 
 use std::fmt::{self, Display, Debug};
 use std::iter::FromIterator;
-use std::ops::Deref;
+use std::ops::{Deref, DerefMut};
 use std::{mem, ptr, slice, vec};
 
 use serialize::{Encodable, Decodable, Encoder, Decoder};
@@ -103,6 +103,12 @@ impl<T: ?Sized> Deref for P<T> {
     }
 }
 
+impl<T: ?Sized> DerefMut for P<T> {
+    fn deref_mut(&mut self) -> &mut T {
+        &mut *self.ptr
+    }
+}
+
 impl<T: 'static + Clone> Clone for P<T> {
     fn clone(&self) -> P<T> {
         P((**self).clone())