Auto merge of #38748 - alexcrichton:upload, r=brson

travis: Start uploading artifacts on commits

This commit starts adding the infrastructure for uploading release artifacts
from AppVeyor/Travis on each commit. The idea is that eventually we'll upload a
full release to AppVeyor/Travis in accordance with plans [outlined earlier].

Right now this configures Travis/Appveyor to upload all tarballs in the `dist`
directory, and various images are updated to actually produce tarballs in these
directories. These are nowhere near ready to be actual release artifacts, but
this should allow us to play around with it and test it out. Once this commit
lands we should start seeing artifacts uploaded on each commit.

[outlined earlier]: https://internals.rust-lang.org/t/rust-ci-release-infrastructure-changes/4489

Author: bors

.travis.yml

@@ -13,20 +13,20 @@ osx_image: xcode8.2
 matrix:
   include:
     # Linux builders, all docker images
-    - env: IMAGE=arm-android
-    - env: IMAGE=cross
-    - env: IMAGE=dist-arm-unknown-linux-gnueabi
-    - env: IMAGE=dist-x86_64-unknown-freebsd
-    - env: IMAGE=i686-gnu
+    - env: IMAGE=arm-android DEPLOY=1
+    - env: IMAGE=cross DEPLOY=1
+    - env: IMAGE=dist-arm-unknown-linux-gnueabi DEPLOY=1
+    - env: IMAGE=dist-x86_64-unknown-freebsd DEPLOY=1
+    - env: IMAGE=i686-gnu DEPLOY=1
     - env: IMAGE=i686-gnu-nopt
-    - env: IMAGE=x86_64-gnu
+    - env: IMAGE=x86_64-gnu DEPLOY=1
     - env: IMAGE=x86_64-gnu-full-bootstrap
     - env: IMAGE=x86_64-gnu-aux
     - env: IMAGE=x86_64-gnu-debug
     - env: IMAGE=x86_64-gnu-nopt
     - env: IMAGE=x86_64-gnu-make
     - env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1 RUST_BACKTRACE=1
-    - env: IMAGE=x86_64-musl
+    - env: IMAGE=x86_64-musl DEPLOY=1
     - env: IMAGE=x86_64-gnu-distcheck
 
     # OSX builders
@@ -39,9 +39,10 @@ matrix:
         curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 |
           tar xJf - -C /usr/local/bin --strip-components=1
     - env: >
-        RUST_CHECK_TARGET=check
+        SCRIPT="./x.py test && ./x.py dist"
         RUST_CONFIGURE_ARGS=--build=i686-apple-darwin
         SRC=.
+        DEPLOY=1
       os: osx
       install: *osx_install_sccache
     - env: >
@@ -51,9 +52,10 @@ matrix:
       os: osx
       install: *osx_install_sccache
     - env: >
-        RUST_CHECK_TARGET=
+        RUST_CHECK_TARGET=dist
         RUST_CONFIGURE_ARGS=--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios
         SRC=.
+        DEPLOY=1
       os: osx
       install: *osx_install_sccache
 
@@ -91,3 +93,29 @@ notifications:
 cache:
   directories:
     - $HOME/docker
+
+before_deploy:
+  - mkdir -p deploy/$TRAVIS_COMMIT
+  - >
+      if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+          cp build/dist/*.tar.gz deploy/$TRAVIS_COMMIT &&
+          find "deploy/$TRAVIS_COMMIT" -maxdepth 1 -type f -exec sh -c 'shasum -a 256 -b "{}" > "{}.sha256"' \;;
+      else
+          cp obj/build/dist/*.tar.gz deploy/$TRAVIS_COMMIT &&
+          find "deploy/$TRAVIS_COMMIT" -maxdepth 1 -type f -exec sh -c 'sha256sum -b "{}" > "{}.sha256"' \;;
+      fi
+
+deploy:
+  - provider: s3
+    bucket: rust-lang-ci
+    skip_cleanup: true
+    local_dir: deploy
+    upload_dir: rustc-builds
+    acl: public_read
+    region: us-east-1
+    access_key_id: AKIAIPQVNYF2T3DTYIWQ
+    secret_access_key:
+      secure: "FBqDqOTeIPMu6v/WYPf4CFSlh9rLRZGKVtpLa5KkyuOhXRTrnEzBduEtS8/FMIxdQImvurhSvxWvqRybMOi4qoVfjMqqpHAI7uBbidbrvAcJoHNsx6BgUNVCIoH6a0UsAjTUtm6/YPIpzbHoLZXPL0GrHPMk6Mu04qVSmcYNWn4="
+    on:
+      branch: auto
+      condition: $DEPLOY = 1

appveyor.yml

@@ -8,10 +8,12 @@ environment:
   # 32/64 bit MSVC
   - MSYS_BITS: 64
     RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
-    RUST_CHECK_TARGET: check
+    SCRIPT: python x.py test && python x.py dist
+    DEPLOY: 1
   - MSYS_BITS: 32
     RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
-    RUST_CHECK_TARGET: check
+    SCRIPT: python x.py test && python x.py dist
+    DEPLOY: 1
 
   # MSVC makefiles
   - MSYS_BITS: 64
@@ -50,10 +52,11 @@ environment:
   # too long on appveyor and this is tested by rustbuild below.
   - MSYS_BITS: 32
     RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
-    RUST_CHECK_TARGET: check
+    SCRIPT: python x.py test && python x.py dist
     MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
     MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
     MINGW_DIR: mingw32
+    DEPLOY: 1
 
   - MSYS_BITS: 32
     RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --disable-rustbuild
@@ -63,11 +66,12 @@ environment:
     MINGW_DIR: mingw32
 
   - MSYS_BITS: 64
-    RUST_CHECK_TARGET: check
+    SCRIPT: python x.py test && python x.py dist
     RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
     MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
     MINGW_ARCHIVE: x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z
     MINGW_DIR: mingw64
+    DEPLOY: 1
 
 clone_depth: 1
 build: false
@@ -123,6 +127,32 @@ branches:
   only:
     - auto
 
+before_deploy:
+  - ps: |
+        New-Item -Path deploy -ItemType directory
+        Get-ChildItem -Path build\dist -Filter '*.tar.gz' | Move-Item -Destination deploy
+        Get-FileHash .\deploy\* | ForEach-Object {
+          [io.file]::WriteAllText($_.Path + ".sha256", $_.Hash.ToLower() + "`n")
+        }
+        Get-ChildItem -Path deploy | Foreach-Object {
+          Push-AppveyorArtifact $_.FullName -FileName ${env:APPVEYOR_REPO_COMMIT}/$_
+        }
+
+deploy:
+  - provider: S3
+    skip_cleanup: true
+    access_key_id: AKIAIPQVNYF2T3DTYIWQ
+    secret_access_key:
+      secure: +11jsUNFTQ9dq5Ad1i2+PeUJaXluFJ0zIJAXESE1dFT3Kdjku4/eDdgyjgsB6GnV
+    bucket: rust-lang-ci
+    set_public: true
+    region: us-east-1
+    artifact: /.*\.(tar.gz|sha256)/
+    folder: rustc-builds
+    on:
+      branch: auto
+      DEPLOY: 1
+
 # init:
 #   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
 # on_finish:

src/bootstrap/check.rs

@@ -457,6 +457,8 @@ fn krate_android(build: &Build,
 
         let output = output(Command::new("adb").arg("shell").arg(&program));
         println!("{}", output);
+
+        t!(fs::create_dir_all(build.out.join("tmp")));
         build.run(Command::new("adb")
                           .arg("pull")
                           .arg(&log)
@@ -516,6 +518,7 @@ pub fn android_copy_libs(build: &Build,
     }
 
     println!("Android copy libs to emulator ({})", target);
+    build.run(Command::new("adb").arg("wait-for-device"));
     build.run(Command::new("adb").arg("remount"));
     build.run(Command::new("adb").args(&["shell", "rm", "-r", ADB_TEST_DIR]));
     build.run(Command::new("adb").args(&["shell", "mkdir", ADB_TEST_DIR]));

src/bootstrap/compile.rs

@@ -16,7 +16,6 @@
 //! compiler. This module is also responsible for assembling the sysroot as it
 //! goes along from the output of the previous stage.
 
-use std::cmp;
 use std::collections::HashMap;
 use std::fs::{self, File};
 use std::path::{Path, PathBuf};
@@ -59,7 +58,7 @@ pub fn std(build: &Build, target: &str, compiler: &Compiler) {
     }
 
     build.run(&mut cargo);
-    update_mtime(&libstd_stamp(build, &compiler, target));
+    update_mtime(build, &libstd_stamp(build, &compiler, target));
 }
 
 /// Link all libstd rlibs/dylibs into the sysroot location.
@@ -145,7 +144,7 @@ pub fn test(build: &Build, target: &str, compiler: &Compiler) {
     cargo.arg("--manifest-path")
          .arg(build.src.join("src/rustc/test_shim/Cargo.toml"));
     build.run(&mut cargo);
-    update_mtime(&libtest_stamp(build, compiler, target));
+    update_mtime(build, &libtest_stamp(build, compiler, target));
 }
 
 /// Same as `std_link`, only for libtest
@@ -390,26 +389,39 @@ pub fn tool(build: &Build, stage: u32, host: &str, tool: &str) {
 }
 
 /// Updates the mtime of a stamp file if necessary, only changing it if it's
-/// older than some other file in the same directory.
+/// older than some other library file in the same directory.
 ///
 /// We don't know what file Cargo is going to output (because there's a hash in
 /// the file name) but we know where it's going to put it. We use this helper to
 /// detect changes to that output file by looking at the modification time for
 /// all files in a directory and updating the stamp if any are newer.
-fn update_mtime(path: &Path) {
-    let mut max = None;
-    if let Ok(entries) = path.parent().unwrap().join("deps").read_dir() {
-        for entry in entries.map(|e| t!(e)) {
-            if t!(entry.file_type()).is_file() {
-                let meta = t!(entry.metadata());
-                let time = FileTime::from_last_modification_time(&meta);
-                max = cmp::max(max, Some(time));
-            }
-        }
-    }
-
-    if !max.is_none() && max <= Some(mtime(path)) {
-        return
+///
+/// Note that we only consider Rust libraries as that's what we're interested in
+/// propagating changes from. Files like executables are tracked elsewhere.
+fn update_mtime(build: &Build, path: &Path) {
+    let entries = match path.parent().unwrap().join("deps").read_dir() {
+        Ok(entries) => entries,
+        Err(_) => return,
+    };
+    let files = entries.map(|e| t!(e)).filter(|e| t!(e.file_type()).is_file());
+    let files = files.filter(|e| {
+        let filename = e.file_name();
+        let filename = filename.to_str().unwrap();
+        filename.ends_with(".rlib") ||
+            filename.ends_with(".lib") ||
+            is_dylib(&filename)
+    });
+    let max = files.max_by_key(|entry| {
+        let meta = t!(entry.metadata());
+        FileTime::from_last_modification_time(&meta)
+    });
+    let max = match max {
+        Some(max) => max,
+        None => return,
+    };
+
+    if mtime(&max.path()) > mtime(path) {
+        build.verbose(&format!("updating {:?} as {:?} changed", path, max.path()));
+        t!(File::create(path));
     }
-    t!(File::create(path));
 }

src/bootstrap/dist.rs

@@ -357,14 +357,12 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) {
 pub fn rust_src(build: &Build) {
     println!("Dist src");
 
-    let plain_name = format!("rustc-{}-src", package_vers(build));
     let name = format!("rust-src-{}", package_vers(build));
     let image = tmpdir(build).join(format!("{}-image", name));
     let _ = fs::remove_dir_all(&image);
 
     let dst = image.join("lib/rustlib/src");
     let dst_src = dst.join("rust");
-    let plain_dst_src = dst.join(&plain_name);
     t!(fs::create_dir_all(&dst_src));
 
     // This is the set of root paths which will become part of the source package
@@ -444,7 +442,11 @@ pub fn rust_src(build: &Build) {
     build.run(&mut cmd);
 
     // Rename directory, so that root folder of tarball has the correct name
-    t!(fs::rename(&dst_src, &plain_dst_src));
+    let plain_name = format!("rustc-{}-src", package_vers(build));
+    let plain_dst_src = tmpdir(build).join(&plain_name);
+    let _ = fs::remove_dir_all(&plain_dst_src);
+    t!(fs::create_dir_all(&plain_dst_src));
+    cp_r(&dst_src, &plain_dst_src);
 
     // Create the version file
     write_file(&plain_dst_src.join("version"), build.version.as_bytes());
@@ -453,10 +455,11 @@ pub fn rust_src(build: &Build) {
     let mut cmd = Command::new("tar");
     cmd.arg("-czf").arg(sanitize_sh(&rust_src_location(build)))
        .arg(&plain_name)
-       .current_dir(&dst);
+       .current_dir(tmpdir(build));
     build.run(&mut cmd);
 
     t!(fs::remove_dir_all(&image));
+    t!(fs::remove_dir_all(&plain_dst_src));
 }
 
 fn install(src: &Path, dstdir: &Path, perms: u32) {

src/bootstrap/flags.rs

@@ -67,6 +67,7 @@ pub enum Subcommand {
     },
     Clean,
     Dist {
+        paths: Vec<PathBuf>,
         install: bool,
     },
 }
@@ -249,6 +250,7 @@ To learn more about a subcommand, run `./x.py <command> -h`
                 opts.optflag("", "install", "run installer as well");
                 m = parse(&opts);
                 Subcommand::Dist {
+                    paths: remaining_as_path(&m),
                     install: m.opt_present("install"),
                 }
             }

src/bootstrap/step.rs

@@ -615,7 +615,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .default(true)
          .only_host_build(true)
          .run(move |s| dist::analysis(build, &s.compiler(), s.target));
-    rules.dist("install", "src")
+    rules.dist("install", "path/to/nowhere")
          .dep(|s| s.name("default:dist"))
          .run(move |s| install::install(build, s.stage, s.target));
 
@@ -932,11 +932,11 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd?
             Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]),
             Subcommand::Test { ref paths, test_args: _ } => (Kind::Test, &paths[..]),
             Subcommand::Bench { ref paths, test_args: _ } => (Kind::Bench, &paths[..]),
-            Subcommand::Dist { install } => {
+            Subcommand::Dist { ref paths, install } => {
                 if install {
                     return vec![self.sbuild.name("install")]
                 } else {
-                    (Kind::Dist, &[][..])
+                    (Kind::Dist, &paths[..])
                 }
             }
             Subcommand::Clean => panic!(),

src/ci/docker/arm-android/Dockerfile

@@ -48,4 +48,9 @@ ENV RUST_CONFIGURE_ARGS \
       --armv7-linux-androideabi-ndk=/android/ndk-arm-9 \
       --i686-linux-android-ndk=/android/ndk-x86-9 \
       --aarch64-linux-android-ndk=/android/ndk-aarch64
-ENV XPY_CHECK test --target arm-linux-androideabi
+
+# Just a smoke test in dist to see if this works for now, we should expand this
+# to all the targets above eventually.
+ENV SCRIPT \
+  python2.7 ../x.py test --target arm-linux-androideabi && \
+  python2.7 ../x.py dist --target arm-linux-androideabi

src/ci/docker/arm-android/start-emulator.sh

@@ -13,5 +13,4 @@ set -ex
 ANDROID_EMULATOR_FORCE_32BIT=true \
   nohup nohup emulator @arm-18 -no-window -partition-size 2047 \
   0<&- &>/dev/null &
-adb wait-for-device
 exec "$@"

src/ci/docker/cross/Dockerfile

@@ -56,7 +56,12 @@ ENV TARGETS=$TARGETS,wasm32-unknown-emscripten
 ENV RUST_CONFIGURE_ARGS \
       --target=$TARGETS \
       --enable-rustbuild
-ENV RUST_CHECK_TARGET ""
+
+# Just a smoke test in dist to see if this works for now, we should expand this
+# to all the targets above eventually.
+ENV SCRIPT \
+  python2.7 ../x.py build && \
+  python2.7 ../x.py dist --target wasm32-unknown-emscripten
 
 ENV AR_s390x_unknown_linux_gnu=s390x-linux-gnu-ar \
     CC_s390x_unknown_linux_gnu=s390x-linux-gnu-gcc \

src/ci/docker/dist-arm-unknown-linux-gnueabi/Dockerfile

@@ -24,7 +24,7 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 ENV RUST_CONFIGURE_ARGS --host=arm-unknown-linux-gnueabi
-ENV XPY_RUN \
-      dist \
-      --host arm-unknown-linux-gnueabi \
-      --target arm-unknown-linux-gnueabi
+ENV SCRIPT \
+      python2.7 ../x.py dist \
+        --host arm-unknown-linux-gnueabi \
+        --target arm-unknown-linux-gnueabi

src/ci/docker/dist-x86_64-unknown-freebsd/Dockerfile

@@ -32,7 +32,7 @@ ENV \
     CXX_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-g++
 
 ENV RUST_CONFIGURE_ARGS --host=x86_64-unknown-freebsd
-ENV XPY_RUN \
-      dist \
-      --host x86_64-unknown-freebsd \
-      --target x86_64-unknown-freebsd
+ENV SCRIPT \
+      python2.7 ../x.py dist \
+        --host x86_64-unknown-freebsd \
+        --target x86_64-unknown-freebsd

src/ci/docker/i686-gnu/Dockerfile

@@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
-ENV RUST_CHECK_TARGET check
+ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist