Auto merge of #41600 - ranma42:xz-dist, r=alexcrichton

Generate XZ-compressed tarballs

Integrate the new `rust-installer` and extend manifests with keys for xz-compressed tarballs.

One of the steps required for #21724

Author: bors

.travis.yml

@@ -95,7 +95,9 @@ matrix:
         MACOSX_DEPLOYMENT_TARGET=10.7
       os: osx
       osx_image: xcode7
-      install: *osx_install_sccache
+      install:
+        - brew install p7zip
+        - *osx_install_sccache
     - env: >
         RUST_CHECK_TARGET=dist
         RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-extended --enable-sanitizers"
@@ -106,7 +108,9 @@ matrix:
         MACOSX_DEPLOYMENT_TARGET=10.7
       os: osx
       osx_image: xcode7
-      install: *osx_install_sccache
+      install:
+        - brew install p7zip
+        - *osx_install_sccache
 
     # "alternate" deployments, these are "nightlies" but don't have assertions
     # turned on, they're deployed to a different location primarily for projects
@@ -123,7 +127,9 @@ matrix:
         MACOSX_DEPLOYMENT_TARGET=10.7
       os: osx
       osx_image: xcode7
-      install: *osx_install_sccache
+      install:
+        - brew install p7zip
+        - *osx_install_sccache
 
 env:
   global:

src/ci/docker/cross/Dockerfile

@@ -10,6 +10,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   git \
   cmake \
   sudo \
+  p7zip-full \
   xz-utils \
   zlib1g-dev \
   g++-arm-linux-gnueabi \

src/ci/docker/dist-aarch64-linux/Dockerfile

@@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   texinfo \
   wget \
+  p7zip-full \
   xz-utils \
   libssl-dev \
   pkg-config

src/ci/docker/dist-android/Dockerfile

@@ -16,6 +16,7 @@ RUN dpkg --add-architecture i386 && \
   openjdk-9-jre \
   sudo \
   libstdc++6:i386 \
+  p7zip-full \
   xz-utils \
   libssl-dev \
   pkg-config

src/ci/docker/dist-arm-linux/Dockerfile

@@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   texinfo \
   wget \
+  p7zip-full \
   xz-utils \
   libssl-dev \
   pkg-config

src/ci/docker/dist-armhf-linux/Dockerfile

@@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   texinfo \
   wget \
+  p7zip-full \
   xz-utils \
   libssl-dev \
   pkg-config

src/ci/docker/dist-armv7-linux/Dockerfile

@@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   texinfo \
   wget \
+  p7zip-full \
   xz-utils \
   libssl-dev \
   pkg-config

src/ci/docker/dist-fuchsia/Dockerfile

@@ -11,6 +11,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   git \
   sudo \
   bzip2 \
+  p7zip-full \
   xz-utils \
   swig \
   libedit-dev \

src/ci/docker/dist-i586-gnu-i686-musl/Dockerfile

@@ -9,6 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   python2.7 \
   git \
   cmake \
+  p7zip-full \
   xz-utils \
   sudo \
   gdb \

src/ci/docker/dist-i686-freebsd/Dockerfile

@@ -11,6 +11,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   sudo \
   bzip2 \
+  p7zip-full \
   xz-utils \
   wget \
   libssl-dev \

src/ci/docker/dist-mips-linux/Dockerfile

@@ -11,6 +11,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   sudo \
   gdb \
+  p7zip-full \
   xz-utils \
   g++-mips-linux-gnu \
   libssl-dev \

src/ci/docker/dist-mips64-linux/Dockerfile

@@ -11,6 +11,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   sudo \
   gdb \
+  p7zip-full \
   xz-utils \
   g++-mips64-linux-gnuabi64 \
   libssl-dev \

src/ci/docker/dist-mips64el-linux/Dockerfile

@@ -11,6 +11,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   sudo \
   gdb \
+  p7zip-full \
   xz-utils \
   g++-mips64el-linux-gnuabi64 \
   libssl-dev \

src/ci/docker/dist-mipsel-linux/Dockerfile

@@ -11,6 +11,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   sudo \
   gdb \
+  p7zip-full \
   xz-utils \
   g++-mipsel-linux-gnu \
   libssl-dev \

src/ci/docker/dist-powerpc-linux/Dockerfile

@@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   texinfo \
   wget \
+  p7zip-full \
   xz-utils \
   libssl-dev \
   pkg-config

src/ci/docker/dist-powerpc64-linux/Dockerfile

@@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   texinfo \
   wget \
+  p7zip-full \
   xz-utils \
   libssl-dev \
  pkg-config

src/ci/docker/dist-powerpc64le-linux/Dockerfile

@@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   texinfo \
   wget \
+  p7zip-full \
   xz-utils \
   libssl-dev \
  pkg-config

src/ci/docker/dist-s390x-linux/Dockerfile

@@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   texinfo \
   wget \
+  p7zip-full \
   xz-utils \
   libssl-dev \
   pkg-config

src/ci/docker/dist-x86_64-freebsd/Dockerfile

@@ -11,6 +11,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   sudo \
   bzip2 \
+  p7zip-full \
   xz-utils \
   wget \
   libssl-dev \

src/ci/docker/dist-x86_64-musl/Dockerfile

@@ -9,6 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   python2.7 \
   git \
   cmake \
+  p7zip-full \
   xz-utils \
   sudo \
   gdb \

src/ci/docker/dist-x86_64-netbsd/Dockerfile

@@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   sudo \
   texinfo \
   wget \
+  p7zip-full \
   xz-utils \
   libssl-dev \
   pkg-config

src/ci/docker/x86_64-gnu-distcheck/Dockerfile

@@ -11,6 +11,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   sudo \
   gdb \
+  p7zip-full \
   xz-utils \
   libssl-dev \
   pkg-config

src/rust-installer

@@ -116,10 +116,26 @@ struct Target {
     available: bool,
     url: Option<String>,
     hash: Option<String>,
+    xz_url: Option<String>,
+    xz_hash: Option<String>,
     components: Option<Vec<Component>>,
     extensions: Option<Vec<Component>>,
 }
 
+impl Target {
+    fn unavailable() -> Target {
+        Target {
+            available: false,
+            url: None,
+            hash: None,
+            xz_url: None,
+            xz_hash: None,
+            components: None,
+            extensions: None,
+        }
+    }
+}
+
 #[derive(RustcEncodable)]
 struct Component {
     pkg: String,
@@ -242,16 +258,12 @@ impl Builder {
             let digest = match self.digests.remove(&filename) {
                 Some(digest) => digest,
                 None => {
-                    pkg.target.insert(host.to_string(), Target {
-                        available: false,
-                        url: None,
-                        hash: None,
-                        components: None,
-                        extensions: None,
-                    });
+                    pkg.target.insert(host.to_string(), Target::unavailable());
                     continue
                 }
             };
+            let xz_filename = filename.replace(".tar.gz", ".tar.xz");
+            let xz_digest = self.digests.remove(&xz_filename);
             let mut components = Vec::new();
             let mut extensions = Vec::new();
 
@@ -293,8 +305,10 @@ impl Builder {
 
             pkg.target.insert(host.to_string(), Target {
                 available: true,
-                url: Some(self.url("rust", host)),
+                url: Some(self.url(&filename)),
                 hash: Some(digest),
+                xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
+                xz_hash: xz_digest,
                 components: Some(components),
                 extensions: Some(extensions),
             });
@@ -312,21 +326,17 @@ impl Builder {
             let filename = self.filename(pkgname, name);
             let digest = match self.digests.remove(&filename) {
                 Some(digest) => digest,
-                None => {
-                    return (name.to_string(), Target {
-                        available: false,
-                        url: None,
-                        hash: None,
-                        components: None,
-                        extensions: None,
-                    })
-                }
+                None => return (name.to_string(), Target::unavailable()),
             };
+            let xz_filename = filename.replace(".tar.gz", ".tar.xz");
+            let xz_digest = self.digests.remove(&xz_filename);
 
             (name.to_string(), Target {
                 available: true,
-                url: Some(self.url(pkgname, name)),
+                url: Some(self.url(&filename)),
                 hash: Some(digest),
+                xz_url: xz_digest.as_ref().map(|_| self.url(&xz_filename)),
+                xz_hash: xz_digest,
                 components: None,
                 extensions: None,
             })
@@ -338,11 +348,11 @@ impl Builder {
         });
     }
 
-    fn url(&self, component: &str, target: &str) -> String {
+    fn url(&self, filename: &str) -> String {
         format!("{}/{}/{}",
                 self.s3_address,
                 self.date,
-                self.filename(component, target))
+                filename)
     }
 
     fn filename(&self, component: &str, target: &str) -> String {