Auto merge of #1871 - jtescher:imutable-s3-uploads, r=smarnach

Upload all content to s3 as public and immutable

Adds cache control header with value `public,max-age=31536000,immutable` to all future crate and README uploads.

Fixes #1826

Author: bors

src/bin/render-readmes.rs

@@ -131,7 +131,14 @@ fn main() {
                 let readme_path = format!("readmes/{0}/{0}-{1}.html", krate_name, version.num);
                 config
                     .uploader
-                    .upload(&client, &readme_path, content, content_length, "text/html")
+                    .upload(
+                        &client,
+                        &readme_path,
+                        content,
+                        content_length,
+                        "text/html",
+                        None,
+                    )
                     .unwrap_or_else(|_| {
                         panic!(
                             "[{}-{}] Couldn't upload file to S3",

src/s3/lib.rs

@@ -45,6 +45,7 @@ impl Bucket {
         content: R,
         content_length: u64,
         content_type: &str,
+        extra_headers: Option<header::HeaderMap>,
     ) -> reqwest::Result<reqwest::Response> {
         let path = if path.starts_with('/') {
             &path[1..]
@@ -60,6 +61,7 @@ impl Bucket {
             .header(header::AUTHORIZATION, auth)
             .header(header::CONTENT_TYPE, content_type)
             .header(header::DATE, date)
+            .headers(extra_headers.unwrap_or_else(header::HeaderMap::new))
             .body(reqwest::Body::sized(content, content_length))
             .send()?
             .error_for_status()

src/tasks/dump_db.rs

@@ -157,6 +157,7 @@ impl DumpTarball {
                 tarfile,
                 content_length,
                 "application/gzip",
+                None,
             )
             .map_err(std_error_no_send)?;
         Ok(())

src/tests/record.rs

@@ -272,7 +272,7 @@ fn replay_http(
     mut exchange: Exchange,
     stdout: &mut dyn Write,
 ) -> Box<dyn Future<Item = hyper::Response<hyper::Body>, Error = hyper::Error> + Send> {
-    static IGNORED_HEADERS: &[&str] = &["authorization", "date", "user-agent"];
+    static IGNORED_HEADERS: &[&str] = &["authorization", "date", "user-agent", "cache-control"];
 
     assert_eq!(req.uri().to_string(), exchange.request.uri);
     assert_eq!(req.method().to_string(), exchange.request.method);

src/uploaders.rs

@@ -1,6 +1,7 @@
 use conduit::Request;
 use flate2::read::GzDecoder;
 use openssl::hash::{Hasher, MessageDigest};
+use reqwest::header;
 
 use crate::util::LimitErrorReader;
 use crate::util::{human, internal, CargoResult, ChainError, Maximums};
@@ -13,6 +14,8 @@ use std::sync::Arc;
 use crate::middleware::app::RequestApp;
 use crate::models::Crate;
 
+pub const CACHE_CONTROL_IMMUTABLE: &str = "public,max-age=31536000,immutable";
+
 #[derive(Clone, Debug)]
 pub enum Uploader {
     /// For production usage, uploads and redirects to s3.
@@ -70,13 +73,13 @@ impl Uploader {
         }
     }
 
-    /// Returns the interna path of an uploaded crate's version archive.
+    /// Returns the internal path of an uploaded crate's version archive.
     fn crate_path(name: &str, version: &str) -> String {
         // No slash in front so we can use join
         format!("crates/{}/{}-{}.crate", name, name, version)
     }
 
-    /// Returns the interna path of an uploaded crate's version readme.
+    /// Returns the internal path of an uploaded crate's version readme.
     fn readme_path(name: &str, version: &str) -> String {
         format!("readmes/{}/{}-{}.html", name, name, version)
     }
@@ -91,11 +94,19 @@ impl Uploader {
         mut content: R,
         content_length: u64,
         content_type: &str,
+        extra_headers: Option<header::HeaderMap>,
     ) -> CargoResult<Option<String>> {
         match *self {
             Uploader::S3 { ref bucket, .. } => {
                 bucket
-                    .put(client, path, content, content_length, content_type)
+                    .put(
+                        client,
+                        path,
+                        content,
+                        content_length,
+                        content_type,
+                        extra_headers,
+                    )
                     .map_err(|e| internal(&format_args!("failed to upload to S3: {}", e)))?;
                 Ok(Some(String::from(path)))
             }
@@ -126,12 +137,18 @@ impl Uploader {
         let checksum = hash(&body);
         let content_length = body.len() as u64;
         let content = Cursor::new(body);
+        let mut extra_headers = header::HeaderMap::new();
+        extra_headers.insert(
+            header::CACHE_CONTROL,
+            CACHE_CONTROL_IMMUTABLE.parse().unwrap(),
+        );
         self.upload(
             app.http_client(),
             &path,
             content,
             content_length,
             "application/x-tar",
+            Some(extra_headers),
         )?;
         Ok(checksum)
     }
@@ -146,7 +163,14 @@ impl Uploader {
         let path = Uploader::readme_path(crate_name, vers);
         let content_length = readme.len() as u64;
         let content = Cursor::new(readme);
-        self.upload(http_client, &path, content, content_length, "text/html")?;
+        self.upload(
+            http_client,
+            &path,
+            content,
+            content_length,
+            "text/html",
+            None,
+        )?;
         Ok(())
     }
 }