Add support for test suites emulated in QEMU

This commit adds support to the build system to execute test suites that cannot
run natively but can instead run inside of a QEMU emulator. A proof-of-concept
builder was added for the `arm-unknown-linux-gnueabihf` target to show off how
this might work.

In general the architecture is to have a server running inside of the emulator
which a local client connects to. The protocol between the server/client
supports compiling tests on the host and running them on the target inside the
emulator.

Closes #33114

Author: alexcrichton

.travis.yml

@@ -13,6 +13,7 @@ matrix:
   include:
     # Linux builders, all docker images
     - env: IMAGE=android DEPLOY=1
+    - env: IMAGE=armhf-gnu
     - env: IMAGE=cross DEPLOY=1
     - env: IMAGE=linux-tested-targets DEPLOY=1
     - env: IMAGE=dist-arm-linux DEPLOY=1

configure

@@ -684,6 +684,7 @@ valopt musl-root-arm "" "arm-unknown-linux-musleabi install directory"
 valopt musl-root-armhf "" "arm-unknown-linux-musleabihf install directory"
 valopt musl-root-armv7 "" "armv7-unknown-linux-musleabihf install directory"
 valopt extra-filename "" "Additional data that is hashed and passed to the -C extra-filename flag"
+valopt qemu-armhf-rootfs "" "rootfs in qemu testing, you probably don't want to use this"
 
 if [ -e ${CFG_SRC_DIR}.git ]
 then

src/Cargo.lock

@@ -11,6 +11,8 @@ members = [
   "tools/rustbook",
   "tools/tidy",
   "tools/build-manifest",
+  "tools/qemu-test-client",
+  "tools/qemu-test-server",
 ]
 
 # Curiously, compiletest will segfault if compiled with opt-level=3 on 64-bit

src/Cargo.toml

@@ -26,7 +26,7 @@ use build_helper::output;
 
 use {Build, Compiler, Mode};
 use dist;
-use util::{self, dylib_path, dylib_path_var};
+use util::{self, dylib_path, dylib_path_var, exe};
 
 const ADB_TEST_DIR: &'static str = "/data/tmp";
 
@@ -221,6 +221,12 @@ pub fn compiletest(build: &Build,
            .arg("--llvm-cxxflags").arg("");
     }
 
+    if build.qemu_rootfs(target).is_some() {
+        cmd.arg("--qemu-test-client")
+           .arg(build.tool(&Compiler::new(0, &build.config.build),
+                           "qemu-test-client"));
+    }
+
     // Running a C compiler on MSVC requires a few env vars to be set, to be
     // sure to set them here.
     //
@@ -403,9 +409,9 @@ pub fn krate(build: &Build,
     dylib_path.insert(0, build.sysroot_libdir(&compiler, target));
     cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
 
-    if target.contains("android") {
-        cargo.arg("--no-run");
-    } else if target.contains("emscripten") {
+    if target.contains("android") ||
+       target.contains("emscripten") ||
+       build.qemu_rootfs(target).is_some() {
         cargo.arg("--no-run");
     }
 
@@ -423,6 +429,9 @@ pub fn krate(build: &Build,
     } else if target.contains("emscripten") {
         build.run(&mut cargo);
         krate_emscripten(build, &compiler, target, mode);
+    } else if build.qemu_rootfs(target).is_some() {
+        build.run(&mut cargo);
+        krate_qemu(build, &compiler, target, mode);
     } else {
         cargo.args(&build.flags.cmd.test_args());
         build.run(&mut cargo);
@@ -480,23 +489,46 @@ fn krate_emscripten(build: &Build,
                     compiler: &Compiler,
                     target: &str,
                     mode: Mode) {
-     let mut tests = Vec::new();
-     let out_dir = build.cargo_out(compiler, mode, target);
-     find_tests(&out_dir, target, &mut tests);
-     find_tests(&out_dir.join("deps"), target, &mut tests);
-
-     for test in tests {
-         let test_file_name = test.to_string_lossy().into_owned();
-         println!("running {}", test_file_name);
-         let nodejs = build.config.nodejs.as_ref().expect("nodejs not configured");
-         let mut cmd = Command::new(nodejs);
-         cmd.arg(&test_file_name);
-         if build.config.quiet_tests {
-             cmd.arg("--quiet");
-         }
-         build.run(&mut cmd);
-     }
- }
+    let mut tests = Vec::new();
+    let out_dir = build.cargo_out(compiler, mode, target);
+    find_tests(&out_dir, target, &mut tests);
+    find_tests(&out_dir.join("deps"), target, &mut tests);
+
+    for test in tests {
+        let test_file_name = test.to_string_lossy().into_owned();
+        println!("running {}", test_file_name);
+        let nodejs = build.config.nodejs.as_ref().expect("nodejs not configured");
+        let mut cmd = Command::new(nodejs);
+        cmd.arg(&test_file_name);
+        if build.config.quiet_tests {
+            cmd.arg("--quiet");
+        }
+        build.run(&mut cmd);
+    }
+}
+
+fn krate_qemu(build: &Build,
+              compiler: &Compiler,
+              target: &str,
+              mode: Mode) {
+    let mut tests = Vec::new();
+    let out_dir = build.cargo_out(compiler, mode, target);
+    find_tests(&out_dir, target, &mut tests);
+    find_tests(&out_dir.join("deps"), target, &mut tests);
+
+    let tool = build.tool(&Compiler::new(0, &build.config.build),
+                          "qemu-test-client");
+    for test in tests {
+        let mut cmd = Command::new(&tool);
+        cmd.arg("run")
+           .arg(&test);
+        if build.config.quiet_tests {
+            cmd.arg("--quiet");
+        }
+        cmd.args(&build.flags.cmd.test_args());
+        build.run(&mut cmd);
+    }
+}
 
 
 fn find_tests(dir: &Path,
@@ -516,13 +548,15 @@ fn find_tests(dir: &Path,
     }
 }
 
-pub fn android_copy_libs(build: &Build,
-                         compiler: &Compiler,
-                         target: &str) {
-    if !target.contains("android") {
-        return
+pub fn emulator_copy_libs(build: &Build, compiler: &Compiler, target: &str) {
+    if target.contains("android") {
+        android_copy_libs(build, compiler, target)
+    } else if let Some(s) = build.qemu_rootfs(target) {
+        qemu_copy_libs(build, compiler, target, s)
     }
+}
 
+fn android_copy_libs(build: &Build, compiler: &Compiler, target: &str) {
     println!("Android copy libs to emulator ({})", target);
     build.run(Command::new("adb").arg("wait-for-device"));
     build.run(Command::new("adb").arg("remount"));
@@ -548,6 +582,39 @@ pub fn android_copy_libs(build: &Build,
     }
 }
 
+fn qemu_copy_libs(build: &Build,
+                  compiler: &Compiler,
+                  target: &str,
+                  rootfs: &Path) {
+    println!("QEMU copy libs to emulator ({})", target);
+    assert!(target.starts_with("arm"), "only works with arm for now");
+    t!(fs::create_dir_all(build.out.join("tmp")));
+
+    // Copy our freshly compiled test server over to the rootfs
+    let server = build.cargo_out(compiler, Mode::Tool, target)
+                      .join(exe("qemu-test-server", target));
+    t!(fs::copy(&server, rootfs.join("testd")));
+
+    // Spawn the emulator and wait for it to come online
+    let tool = build.tool(&Compiler::new(0, &build.config.build),
+                          "qemu-test-client");
+    build.run(Command::new(&tool)
+                      .arg("spawn-emulator")
+                      .arg(rootfs)
+                      .arg(build.out.join("tmp")));
+
+    // Push all our dylibs to the emulator
+    for f in t!(build.sysroot_libdir(compiler, target).read_dir()) {
+        let f = t!(f);
+        let name = f.file_name().into_string().unwrap();
+        if util::is_dylib(&name) {
+            build.run(Command::new(&tool)
+                              .arg("push")
+                              .arg(f.path()));
+        }
+    }
+}
+
 /// Run "distcheck", a 'make check' from a tarball
 pub fn distcheck(build: &Build) {
     if build.config.build != "x86_64-unknown-linux-gnu" {

src/bootstrap/check.rs

@@ -382,10 +382,10 @@ fn add_to_sysroot(out_dir: &Path, sysroot_dst: &Path) {
 ///
 /// This will build the specified tool with the specified `host` compiler in
 /// `stage` into the normal cargo output directory.
-pub fn tool(build: &Build, stage: u32, host: &str, tool: &str) {
-    println!("Building stage{} tool {} ({})", stage, tool, host);
+pub fn tool(build: &Build, stage: u32, target: &str, tool: &str) {
+    println!("Building stage{} tool {} ({})", stage, tool, target);
 
-    let compiler = Compiler::new(stage, host);
+    let compiler = Compiler::new(stage, &build.config.build);
 
     // FIXME: need to clear out previous tool and ideally deps, may require
     //        isolating output directories or require a pseudo shim step to
@@ -396,7 +396,7 @@ pub fn tool(build: &Build, stage: u32, host: &str, tool: &str) {
     // let out_dir = build.cargo_out(stage, &host, Mode::Librustc, target);
     // build.clear_if_dirty(&out_dir, &libstd_stamp(build, stage, &host, target));
 
-    let mut cargo = build.cargo(&compiler, Mode::Tool, host, "build");
+    let mut cargo = build.cargo(&compiler, Mode::Tool, target, "build");
     cargo.arg("--manifest-path")
          .arg(build.src.join(format!("src/tools/{}/Cargo.toml", tool)));
 

src/bootstrap/compile.rs

@@ -114,6 +114,7 @@ pub struct Target {
     pub cxx: Option<PathBuf>,
     pub ndk: Option<PathBuf>,
     pub musl_root: Option<PathBuf>,
+    pub qemu_rootfs: Option<PathBuf>,
 }
 
 /// Structure of the `config.toml` file that configuration is read from.
@@ -222,6 +223,7 @@ struct TomlTarget {
     cxx: Option<String>,
     android_ndk: Option<String>,
     musl_root: Option<String>,
+    qemu_rootfs: Option<String>,
 }
 
 impl Config {
@@ -360,6 +362,7 @@ impl Config {
                 target.cxx = cfg.cxx.clone().map(PathBuf::from);
                 target.cc = cfg.cc.clone().map(PathBuf::from);
                 target.musl_root = cfg.musl_root.clone().map(PathBuf::from);
+                target.qemu_rootfs = cfg.qemu_rootfs.clone().map(PathBuf::from);
 
                 config.target_config.insert(triple.clone(), target);
             }
@@ -562,6 +565,12 @@ impl Config {
                                                .map(|s| s.to_string())
                                                .collect();
                 }
+                "CFG_QEMU_ARMHF_ROOTFS" if value.len() > 0 => {
+                    let target = "arm-unknown-linux-gnueabihf".to_string();
+                    let target = self.target_config.entry(target)
+                                     .or_insert(Target::default());
+                    target.qemu_rootfs = Some(parse_configure_path(value));
+                }
                 _ => {}
             }
         }

src/bootstrap/config.rs

@@ -886,6 +886,17 @@ impl Build {
             .map(|p| &**p)
     }
 
+    /// Returns the root of the "rootfs" image that this target will be using,
+    /// if one was configured.
+    ///
+    /// If `Some` is returned then that means that tests for this target are
+    /// emulated with QEMU and binaries will need to be shipped to the emulator.
+    fn qemu_rootfs(&self, target: &str) -> Option<&Path> {
+        self.config.target_config.get(target)
+            .and_then(|t| t.qemu_rootfs.as_ref())
+            .map(|p| &**p)
+    }
+
     /// Path to the python interpreter to use
     fn python(&self) -> &Path {
         self.config.python.as_ref().unwrap()