Upload sources even for binaries and failed builds

Author: jyn514

src/db/add_package.rs

@@ -30,7 +30,7 @@ pub(crate) fn add_package_into_database(
     source_dir: &Path,
     res: &BuildResult,
     default_target: &str,
-    source_files: Option<Value>,
+    source_files: Value,
     doc_targets: Vec<String>,
     registry_data: &ReleaseData,
     has_docs: bool,

src/docbuilder/rustwide_builder.rs

@@ -348,26 +348,23 @@ impl RustwideBuilder {
             .run(|build| {
                 use crate::docbuilder::metadata::BuildTargets;
 
-                let mut files_list = None;
                 let mut has_docs = false;
-                let mut algs = CompressionAlgorithms::default();
                 let mut successful_targets = Vec::new();
                 let metadata = Metadata::from_source_dir(&build.host_source_dir())?;
                 let BuildTargets {
                     default_target,
                     other_targets,
                 } = metadata.targets();
 
-                // Do an initial build and then copy the sources in the database
+                // Store the sources even if the build fails
+                debug!("adding sources into database");
+                let prefix = format!("sources/{}/{}", name, version);
+                let (files_list, mut algs) =
+                    add_path_into_database(&self.storage, &prefix, build.host_source_dir())?;
+
+                // Perform an initial build
                 let res = self.execute_build(default_target, true, &build, &limits, &metadata)?;
                 if res.result.successful {
-                    debug!("adding sources into database");
-                    let prefix = format!("sources/{}/{}", name, version);
-                    let (files, new_algs) =
-                        add_path_into_database(&self.storage, &prefix, build.host_source_dir())?;
-                    files_list = Some(files);
-                    algs.extend(new_algs);
-
                     if let Some(name) = res.cargo_metadata.root().library_name() {
                         let host_target = build.host_target_dir();
                         has_docs = host_target.join("doc").join(name).is_dir();

src/test/fakes.rs

@@ -184,67 +184,64 @@ impl<'a> FakeRelease<'a> {
 
     /// Returns the release_id
     pub(crate) fn create(self) -> Result<i32, Error> {
-        use std::collections::HashSet;
         use std::fs;
         use std::path::Path;
 
         let tempdir = tempfile::Builder::new().prefix("docs.rs-fake").tempdir()?;
         let package = self.package;
         let db = self.db;
+        let mut rustdoc_files = self.rustdoc_files;
+        let storage = self.storage;
 
-        let mut source_meta = None;
-        let mut algs = HashSet::new();
-        if self.build_result.successful {
-            let storage = self.storage.clone();
-            let upload_files = |prefix: &str, files: &[(&str, &[u8])], target: Option<&str>| {
-                let mut path_prefix = tempdir.path().join(prefix);
-                if let Some(target) = target {
-                    path_prefix.push(target);
-                }
-                fs::create_dir(&path_prefix)?;
-
-                for (path, data) in files {
-                    // allow `src/main.rs`
-                    if let Some(parent) = Path::new(path).parent() {
-                        fs::create_dir_all(path_prefix.join(parent))?;
-                    }
-                    let file = path_prefix.join(&path);
-                    log::debug!("writing file {}", file.display());
-                    fs::write(file, data)?;
+        // Upload all source files as rustdoc files
+        // In real life, these would be highlighted HTML, but for testing we just use the files themselves.
+        for (source_path, data) in &self.source_files {
+            if source_path.starts_with("src/") {
+                let updated = ["src", &package.name, &source_path[4..]].join("/");
+                rustdoc_files.push((Box::leak(Box::new(updated)), data));
+            }
+        }
+
+        let upload_files = |prefix: &str, files: &[(&str, &[u8])], target: Option<&str>| {
+            let mut path_prefix = tempdir.path().join(prefix);
+            if let Some(target) = target {
+                path_prefix.push(target);
+            }
+            fs::create_dir(&path_prefix)?;
+
+            for (path, data) in files {
+                // allow `src/main.rs`
+                if let Some(parent) = Path::new(path).parent() {
+                    fs::create_dir_all(path_prefix.join(parent))?;
                 }
+                let file = path_prefix.join(&path);
+                log::debug!("writing file {}", file.display());
+                fs::write(file, data)?;
+            }
 
-                let prefix = format!(
-                    "{}/{}/{}/{}",
-                    prefix,
-                    package.name,
-                    package.version,
-                    target.unwrap_or("")
-                );
-                log::debug!("adding directory {} from {}", prefix, path_prefix.display());
-                crate::db::add_path_into_database(&storage, &prefix, path_prefix)
-            };
+            let prefix = format!(
+                "{}/{}/{}/{}",
+                prefix,
+                package.name,
+                package.version,
+                target.unwrap_or("")
+            );
+            log::debug!("adding directory {} from {}", prefix, path_prefix.display());
+            crate::db::add_path_into_database(&storage, &prefix, path_prefix)
+        };
 
+        let (source_meta, mut algs) = upload_files("source", &self.source_files, None)?;
+        log::debug!("added source files {}", source_meta);
+
+        if self.build_result.successful {
             let index = [&package.name, "index.html"].join("/");
-            let mut rustdoc_files = self.rustdoc_files;
             if package.is_library() && !rustdoc_files.iter().any(|(path, _)| path == &index) {
                 rustdoc_files.push((&index, DEFAULT_CONTENT));
             }
-            for (source_path, data) in &self.source_files {
-                if source_path.starts_with("src/") {
-                    let updated = ["src", &package.name, &source_path[4..]].join("/");
-                    rustdoc_files.push((Box::leak(Box::new(updated)), data));
-                }
-            }
+
             let (rustdoc_meta, new_algs) = upload_files("rustdoc", &rustdoc_files, None)?;
             algs.extend(new_algs);
             log::debug!("added rustdoc files {}", rustdoc_meta);
-            match upload_files("source", &self.source_files, None)? {
-                (json, new_algs) => {
-                    source_meta = Some(json);
-                    algs.extend(new_algs);
-                }
-            }
-            log::debug!("added source files {}", source_meta.as_ref().unwrap());
 
             for target in &package.targets[1..] {
                 let platform = target.src_path.as_ref().unwrap();