Upload build log to S3

Author: Nemo157

src/db/add_package.rs

@@ -175,17 +175,14 @@ pub(crate) fn add_build_into_database(
 ) -> Result<i32> {
     debug!("Adding build into database");
     let rows = conn.query(
-        "INSERT INTO builds (rid, rustc_version,
-                                                    cratesfyi_version,
-                                                    build_status, output)
-                                VALUES ($1, $2, $3, $4, $5)
-                                RETURNING id",
+        "INSERT INTO builds (rid, rustc_version, cratesfyi_version, build_status)
+        VALUES ($1, $2, $3, $4)
+        RETURNING id",
         &[
             &release_id,
             &res.rustc_version,
             &res.docsrs_version,
             &res.successful,
-            &res.build_log,
         ],
     )?;
     Ok(rows[0].get(0))

src/docbuilder/rustwide_builder.rs

@@ -322,7 +322,7 @@ impl RustwideBuilder {
 
         let local_storage = tempfile::Builder::new().prefix("docsrs-docs").tempdir()?;
 
-        let res = build_dir
+        let successful = build_dir
             .build(&self.toolchain, &krate, self.prepare_sandbox(&limits))
             .run(|build| {
                 use docsrs_metadata::BuildTargets;
@@ -414,21 +414,23 @@ impl RustwideBuilder {
                     add_doc_coverage(&mut conn, release_id, doc_coverage)?;
                 }
 
-                add_build_into_database(&mut conn, release_id, &res.result)?;
+                let build_id = add_build_into_database(&mut conn, release_id, &res.result)?;
+                let build_log_path = format!("build-logs/{}/{}.txt", build_id, default_target);
+                self.storage.store_one(build_log_path, res.build_log)?;
 
                 // Some crates.io crate data is mutable, so we proactively update it during a release
                 match self.index.api().get_crate_data(name) {
                     Ok(crate_data) => update_crate_data_in_database(&mut conn, name, &crate_data)?,
                     Err(err) => warn!("{:#?}", err),
                 }
 
-                Ok(res)
+                Ok(res.result.successful)
             })?;
 
         build_dir.purge()?;
         krate.purge_from_cache(&self.workspace)?;
         local_storage.close()?;
-        Ok(res.result.successful)
+        Ok(successful)
     }
 
     fn build_target(
@@ -556,13 +558,13 @@ impl RustwideBuilder {
 
         Ok(FullBuildResult {
             result: BuildResult {
-                build_log: storage.to_string(),
                 rustc_version: self.rustc_version.clone(),
                 docsrs_version: format!("docsrs {}", crate::BUILD_VERSION),
                 successful,
             },
             doc_coverage,
             cargo_metadata,
+            build_log: storage.to_string(),
             target: target.to_string(),
         })
     }
@@ -706,6 +708,7 @@ struct FullBuildResult {
     target: String,
     cargo_metadata: CargoMetadata,
     doc_coverage: Option<DocCoverage>,
+    build_log: String,
 }
 
 #[derive(Clone, Copy)]
@@ -725,6 +728,5 @@ pub(crate) struct DocCoverage {
 pub(crate) struct BuildResult {
     pub(crate) rustc_version: String,
     pub(crate) docsrs_version: String,
-    pub(crate) build_log: String,
     pub(crate) successful: bool,
 }

src/storage/mod.rs

@@ -149,9 +149,6 @@ impl Storage {
 
     // Store all files in `root_dir` into the backend under `prefix`.
     //
-    // If the environment is configured with S3 credentials, this will upload to S3;
-    // otherwise, this will store files in the database.
-    //
     // This returns (map<filename, mime type>, set<compression algorithms>).
     pub(crate) fn store_all(
         &self,
@@ -199,10 +196,36 @@ impl Storage {
         self.store_inner(blobs.into_iter().map(Ok))
     }
 
+    // Store file into the backend at the given path (also used to detect mime type), returns the
+    // chosen compression algorithm
+    pub(crate) fn store_one(
+        &self,
+        path: impl Into<String>,
+        content: impl Into<Vec<u8>>,
+    ) -> Result<CompressionAlgorithm, Error> {
+        let path = path.into();
+        let content = content.into();
+        let alg = CompressionAlgorithm::default();
+        let content = compress(&*content, alg)?;
+        let mime = detect_mime(&path)?.to_owned();
+
+        self.store_inner(std::iter::once(Ok(Blob {
+            path,
+            mime,
+            content,
+            compression: Some(alg),
+            // this field is ignored by the backend
+            date_updated: Utc::now(),
+        })))?;
+
+        Ok(alg)
+    }
+
     fn store_inner(
         &self,
-        mut blobs: impl Iterator<Item = Result<Blob, Error>>,
+        blobs: impl IntoIterator<Item = Result<Blob, Error>>,
     ) -> Result<(), Error> {
+        let mut blobs = blobs.into_iter();
         self.transaction(|trans| {
             loop {
                 let batch: Vec<_> = blobs
@@ -249,13 +272,13 @@ trait StorageTransaction {
     fn complete(self: Box<Self>) -> Result<(), Error>;
 }
 
-fn detect_mime(file_path: &Path) -> Result<&'static str, Error> {
-    let mime = mime_guess::from_path(file_path)
+fn detect_mime(file_path: impl AsRef<Path>) -> Result<&'static str, Error> {
+    let mime = mime_guess::from_path(file_path.as_ref())
         .first_raw()
         .unwrap_or("text/plain");
     Ok(match mime {
         "text/plain" | "text/troff" | "text/x-markdown" | "text/x-rust" | "text/x-toml" => {
-            match file_path.extension().and_then(OsStr::to_str) {
+            match file_path.as_ref().extension().and_then(OsStr::to_str) {
                 Some("md") => "text/markdown",
                 Some("rs") => "text/rust",
                 Some("markdown") => "text/markdown",

src/test/fakes.rs

@@ -32,6 +32,8 @@ pub(crate) struct FakeRelease<'a> {
 }
 
 pub(crate) struct FakeBuild {
+    s3_build_log: Option<String>,
+    db_build_log: Option<String>,
     result: BuildResult,
 }
 
@@ -323,12 +325,13 @@ impl<'a> FakeRelease<'a> {
         if let Some(markdown) = self.readme {
             fs::write(crate_dir.join("README.md"), markdown)?;
         }
+        let default_target = self.default_target.unwrap_or(docsrs_metadata::HOST_TARGET);
         let release_id = crate::db::add_package_into_database(
             &mut db.conn(),
             &package,
             crate_dir,
             last_build_result,
-            self.default_target.unwrap_or("x86_64-unknown-linux-gnu"),
+            default_target,
             source_meta,
             self.doc_targets,
             &self.registry_release_data,
@@ -343,7 +346,7 @@ impl<'a> FakeRelease<'a> {
             &self.registry_crate_data,
         )?;
         for build in &self.builds {
-            build.create(&mut db.conn(), release_id)?;
+            build.create(&mut db.conn(), &*storage, release_id, default_target)?;
         }
         if let Some(coverage) = self.doc_coverage {
             crate::db::add_doc_coverage(&mut db.conn(), release_id, coverage)?;
@@ -384,6 +387,7 @@ impl FakeBuild {
                 rustc_version: rustc_version.into(),
                 ..self.result
             },
+            ..self
         }
     }
 
@@ -393,15 +397,28 @@ impl FakeBuild {
                 docsrs_version: docsrs_version.into(),
                 ..self.result
             },
+            ..self
         }
     }
 
-    pub(crate) fn build_log(self, build_log: impl Into<String>) -> Self {
+    pub(crate) fn s3_build_log(self, build_log: impl Into<String>) -> Self {
         Self {
-            result: BuildResult {
-                build_log: build_log.into(),
-                ..self.result
-            },
+            s3_build_log: Some(build_log.into()),
+            ..self
+        }
+    }
+
+    pub(crate) fn db_build_log(self, build_log: impl Into<String>) -> Self {
+        Self {
+            db_build_log: Some(build_log.into()),
+            ..self
+        }
+    }
+
+    pub(crate) fn no_s3_build_log(self) -> Self {
+        Self {
+            s3_build_log: None,
+            ..self
         }
     }
 
@@ -411,11 +428,30 @@ impl FakeBuild {
                 successful,
                 ..self.result
             },
+            ..self
         }
     }
 
-    fn create(&self, conn: &mut Client, release_id: i32) -> Result<(), Error> {
-        crate::db::add_build_into_database(conn, release_id, &self.result)?;
+    fn create(
+        &self,
+        conn: &mut Client,
+        storage: &Storage,
+        release_id: i32,
+        default_target: &str,
+    ) -> Result<(), Error> {
+        let build_id = crate::db::add_build_into_database(conn, release_id, &self.result)?;
+
+        if let Some(db_build_log) = self.db_build_log.as_deref() {
+            conn.query(
+                "UPDATE builds SET output = $2 WHERE id = $1",
+                &[&build_id, &db_build_log],
+            )?;
+        }
+
+        if let Some(s3_build_log) = self.s3_build_log.as_deref() {
+            let path = format!("build-logs/{}/{}.txt", build_id, default_target);
+            storage.store_one(path, s3_build_log)?;
+        }
 
         Ok(())
     }
@@ -424,10 +460,11 @@ impl FakeBuild {
 impl Default for FakeBuild {
     fn default() -> Self {
         Self {
+            s3_build_log: Some("It works!".into()),
+            db_build_log: None,
             result: BuildResult {
                 rustc_version: "rustc 2.0.0-nightly (000000000 1970-01-01)".into(),
                 docsrs_version: "docs.rs 1.0.0 (000000000 1970-01-01)".into(),
-                build_log: "It works!".into(),
                 successful: true,
             },
         }