Abbreviate some stdout/stderr output in compiletest.

This is intended to prevent the spurious OOM error from
run-pass/rustc-rust-log.rs, by skipping the output in the middle when the
size is over 416 KB, so that the log output will not be overwhelmed.

Author: kennytm

src/Cargo.lock

@@ -11,3 +11,7 @@ getopts = "0.2"
 log = "0.3"
 rustc-serialize = "0.3"
 libc = "0.2"
+
+[target.'cfg(windows)'.dependencies]
+miow = "0.2"
+winapi = "0.2"

src/tools/compiletest/Cargo.toml

@@ -11,10 +11,11 @@
 #![crate_name = "compiletest"]
 
 #![feature(test)]
+#![feature(slice_rotate)]
 
 #![deny(warnings)]
 
-#[cfg(any(target_os = "macos", target_os = "ios"))]
+#[cfg(unix)]
 extern crate libc;
 extern crate test;
 extern crate getopts;
@@ -47,6 +48,7 @@ pub mod runtest;
 pub mod common;
 pub mod errors;
 mod raise_fd_limit;
+mod read2;
 
 fn main() {
     env_logger::init().unwrap();

src/tools/compiletest/src/main.rs

@@ -0,0 +1,208 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// FIXME: This is a complete copy of `cargo/src/cargo/util/read2.rs`
+// Consider unify the read2() in libstd, cargo and this to prevent further code duplication.
+
+pub use self::imp::read2;
+
+#[cfg(not(any(unix, windows)))]
+mod imp {
+    use std::io::{self, Read};
+    use std::process::{ChildStdout, ChildStderr};
+
+    pub fn read2(out_pipe: ChildStdout,
+                 err_pipe: ChildStderr,
+                 data: &mut FnMut(bool, &mut Vec<u8>, bool)) -> io::Result<()> {
+        let mut buffer = Vec::new();
+        out_pipe.read_to_end(&mut buffer)?;
+        data(true, &mut buffer, true);
+        buffer.clear();
+        err_pipe.read_to_end(&mut buffer)?;
+        data(false, &mut buffer, true);
+        Ok(())
+    }
+}
+
+#[cfg(unix)]
+mod imp {
+    use std::io::prelude::*;
+    use std::io;
+    use std::mem;
+    use std::os::unix::prelude::*;
+    use std::process::{ChildStdout, ChildStderr};
+    use libc;
+
+    pub fn read2(mut out_pipe: ChildStdout,
+                 mut err_pipe: ChildStderr,
+                 data: &mut FnMut(bool, &mut Vec<u8>, bool)) -> io::Result<()> {
+        unsafe {
+            libc::fcntl(out_pipe.as_raw_fd(), libc::F_SETFL, libc::O_NONBLOCK);
+            libc::fcntl(err_pipe.as_raw_fd(), libc::F_SETFL, libc::O_NONBLOCK);
+        }
+
+        let mut out_done = false;
+        let mut err_done = false;
+        let mut out = Vec::new();
+        let mut err = Vec::new();
+
+        let mut fds: [libc::pollfd; 2] = unsafe { mem::zeroed() };
+        fds[0].fd = out_pipe.as_raw_fd();
+        fds[0].events = libc::POLLIN;
+        fds[1].fd = err_pipe.as_raw_fd();
+        fds[1].events = libc::POLLIN;
+        loop {
+            // wait for either pipe to become readable using `select`
+            let r = unsafe { libc::poll(fds.as_mut_ptr(), 2, -1) };
+            if r == -1 {
+                let err = io::Error::last_os_error();
+                if err.kind() == io::ErrorKind::Interrupted {
+                    continue
+                }
+                return Err(err)
+            }
+
+            // Read as much as we can from each pipe, ignoring EWOULDBLOCK or
+            // EAGAIN. If we hit EOF, then this will happen because the underlying
+            // reader will return Ok(0), in which case we'll see `Ok` ourselves. In
+            // this case we flip the other fd back into blocking mode and read
+            // whatever's leftover on that file descriptor.
+            let handle = |res: io::Result<_>| {
+                match res {
+                    Ok(_) => Ok(true),
+                    Err(e) => {
+                        if e.kind() == io::ErrorKind::WouldBlock {
+                            Ok(false)
+                        } else {
+                            Err(e)
+                        }
+                    }
+                }
+            };
+            if !out_done && fds[0].revents != 0 && handle(out_pipe.read_to_end(&mut out))? {
+                out_done = true;
+            }
+            data(true, &mut out, out_done);
+            if !err_done && fds[1].revents != 0 && handle(err_pipe.read_to_end(&mut err))? {
+                err_done = true;
+            }
+            data(false, &mut err, err_done);
+
+            if out_done && err_done {
+                return Ok(())
+            }
+        }
+    }
+}
+
+#[cfg(windows)]
+mod imp {
+    extern crate miow;
+    extern crate winapi;
+
+    use std::io;
+    use std::os::windows::prelude::*;
+    use std::process::{ChildStdout, ChildStderr};
+    use std::slice;
+
+    use self::miow::iocp::{CompletionPort, CompletionStatus};
+    use self::miow::pipe::NamedPipe;
+    use self::miow::Overlapped;
+    use self::winapi::ERROR_BROKEN_PIPE;
+
+    struct Pipe<'a> {
+        dst: &'a mut Vec<u8>,
+        overlapped: Overlapped,
+        pipe: NamedPipe,
+        done: bool,
+    }
+
+    pub fn read2(out_pipe: ChildStdout,
+                 err_pipe: ChildStderr,
+                 data: &mut FnMut(bool, &mut Vec<u8>, bool)) -> io::Result<()> {
+        let mut out = Vec::new();
+        let mut err = Vec::new();
+
+        let port = CompletionPort::new(1)?;
+        port.add_handle(0, &out_pipe)?;
+        port.add_handle(1, &err_pipe)?;
+
+        unsafe {
+            let mut out_pipe = Pipe::new(out_pipe, &mut out);
+            let mut err_pipe = Pipe::new(err_pipe, &mut err);
+
+            out_pipe.read()?;
+            err_pipe.read()?;
+
+            let mut status = [CompletionStatus::zero(), CompletionStatus::zero()];
+
+            while !out_pipe.done || !err_pipe.done {
+                for status in port.get_many(&mut status, None)? {
+                    if status.token() == 0 {
+                        out_pipe.complete(status);
+                        data(true, out_pipe.dst, out_pipe.done);
+                        out_pipe.read()?;
+                    } else {
+                        err_pipe.complete(status);
+                        data(false, err_pipe.dst, err_pipe.done);
+                        err_pipe.read()?;
+                    }
+                }
+            }
+
+            Ok(())
+        }
+    }
+
+    impl<'a> Pipe<'a> {
+        unsafe fn new<P: IntoRawHandle>(p: P, dst: &'a mut Vec<u8>) -> Pipe<'a> {
+            Pipe {
+                dst: dst,
+                pipe: NamedPipe::from_raw_handle(p.into_raw_handle()),
+                overlapped: Overlapped::zero(),
+                done: false,
+            }
+        }
+
+        unsafe fn read(&mut self) -> io::Result<()> {
+            let dst = slice_to_end(self.dst);
+            match self.pipe.read_overlapped(dst, self.overlapped.raw()) {
+                Ok(_) => Ok(()),
+                Err(e) => {
+                    if e.raw_os_error() == Some(ERROR_BROKEN_PIPE as i32) {
+                        self.done = true;
+                        Ok(())
+                    } else {
+                        Err(e)
+                    }
+                }
+            }
+        }
+
+        unsafe fn complete(&mut self, status: &CompletionStatus) {
+            let prev = self.dst.len();
+            self.dst.set_len(prev + status.bytes_transferred() as usize);
+            if status.bytes_transferred() == 0 {
+                self.done = true;
+            }
+        }
+    }
+
+    unsafe fn slice_to_end(v: &mut Vec<u8>) -> &mut [u8] {
+        if v.capacity() == 0 {
+            v.reserve(16);
+        }
+        if v.capacity() == v.len() {
+            v.reserve(1);
+        }
+        slice::from_raw_parts_mut(v.as_mut_ptr().offset(v.len() as isize),
+                                  v.capacity() - v.len())
+    }
+}

src/tools/compiletest/src/read2.rs

@@ -29,7 +29,7 @@ use std::fmt;
 use std::io::prelude::*;
 use std::io::{self, BufReader};
 use std::path::{Path, PathBuf};
-use std::process::{Command, Output, ExitStatus, Stdio};
+use std::process::{Command, Output, ExitStatus, Stdio, Child};
 use std::str;
 
 use extract_gdb_version;
@@ -1344,12 +1344,14 @@ actual:\n\
         if let Some(input) = input {
             child.stdin.as_mut().unwrap().write_all(input.as_bytes()).unwrap();
         }
-        let Output { status, stdout, stderr } = child.wait_with_output().unwrap();
+
+        let Output { status, stdout, stderr } = read2_abbreviated(child)
+            .expect("failed to read output");
 
         let result = ProcRes {
             status,
-            stdout: String::from_utf8(stdout).unwrap(),
-            stderr: String::from_utf8(stderr).unwrap(),
+            stdout: String::from_utf8_lossy(&stdout).into_owned(),
+            stderr: String::from_utf8_lossy(&stderr).into_owned(),
             cmdline,
         };
 
@@ -1634,7 +1636,9 @@ actual:\n\
         cmd.arg("-a").arg("-u");
         cmd.arg(filename);
         cmd.arg("-nobanner");
-        let output = match cmd.output() {
+        cmd.stdout(Stdio::piped());
+        cmd.stderr(Stdio::piped());
+        let output = match cmd.spawn().and_then(read2_abbreviated) {
             Ok(output) => output,
             Err(_) => return,
         };
@@ -2094,6 +2098,8 @@ actual:\n\
 
         let mut cmd = Command::new(make);
         cmd.current_dir(&self.testpaths.file)
+           .stdout(Stdio::piped())
+           .stderr(Stdio::piped())
            .env("TARGET", &self.config.target)
            .env("PYTHON", &self.config.docck_python)
            .env("S", src_root)
@@ -2142,7 +2148,7 @@ actual:\n\
             }
         }
 
-        let output = cmd.output().expect("failed to spawn `make`");
+        let output = cmd.spawn().and_then(read2_abbreviated).expect("failed to spawn `make`");
         if !output.status.success() {
             let res = ProcRes {
                 status: output.status,
@@ -2534,3 +2540,76 @@ fn nocomment_mir_line(line: &str) -> &str {
         line
     }
 }
+
+fn read2_abbreviated(mut child: Child) -> io::Result<Output> {
+    use std::mem::replace;
+    use read2::read2;
+
+    const HEAD_LEN: usize = 160 * 1024;
+    const TAIL_LEN: usize = 256 * 1024;
+
+    enum ProcOutput {
+        Full(Vec<u8>),
+        Abbreviated {
+            head: Vec<u8>,
+            skipped: usize,
+            tail: Box<[u8]>,
+        }
+    }
+
+    impl ProcOutput {
+        fn extend(&mut self, data: &[u8]) {
+            let new_self = match *self {
+                ProcOutput::Full(ref mut bytes) => {
+                    bytes.extend_from_slice(data);
+                    let new_len = bytes.len();
+                    if new_len <= HEAD_LEN + TAIL_LEN {
+                        return;
+                    }
+                    let tail = bytes.split_off(new_len - TAIL_LEN).into_boxed_slice();
+                    let head = replace(bytes, Vec::new());
+                    let skipped = new_len - HEAD_LEN - TAIL_LEN;
+                    ProcOutput::Abbreviated { head, skipped, tail }
+                }
+                ProcOutput::Abbreviated { ref mut skipped, ref mut tail, .. } => {
+                    *skipped += data.len();
+                    if data.len() <= TAIL_LEN {
+                        tail[..data.len()].copy_from_slice(data);
+                        tail.rotate(data.len());
+                    } else {
+                        tail.copy_from_slice(&data[(data.len() - TAIL_LEN)..]);
+                    }
+                    return;
+                }
+            };
+            *self = new_self;
+        }
+
+        fn into_bytes(self) -> Vec<u8> {
+            match self {
+                ProcOutput::Full(bytes) => bytes,
+                ProcOutput::Abbreviated { mut head, skipped, tail } => {
+                    write!(&mut head, "\n\n<<<<<< SKIPPED {} BYTES >>>>>>\n\n", skipped).unwrap();
+                    head.extend_from_slice(&tail);
+                    head
+                }
+            }
+        }
+    }
+
+    let mut stdout = ProcOutput::Full(Vec::new());
+    let mut stderr = ProcOutput::Full(Vec::new());
+
+    drop(child.stdin.take());
+    read2(child.stdout.take().unwrap(), child.stderr.take().unwrap(), &mut |is_stdout, data, _| {
+        if is_stdout { &mut stdout } else { &mut stderr }.extend(data);
+        data.clear();
+    })?;
+    let status = child.wait()?;
+
+    Ok(Output {
+        status,
+        stdout: stdout.into_bytes(),
+        stderr: stderr.into_bytes(),
+    })
+}