delete crates when they are deleted from the index

* using the changes from Byron/crates-index-diff-rs#15
* also upgrades `rustwide` and `git2` since `crates-index-diff`
  the includes release includes an update of `libgit2`
* adaptions due to `rustwide` changes in
  rust-lang/rustwide#65

Author: syphar

Cargo.lock

@@ -28,7 +28,7 @@ regex = "1"
 structopt = "0.3"
 crates-index = { version = "0.18.5", optional = true }
 rayon = { version = "1", optional = true }
-crates-index-diff = "8.0.0"
+crates-index-diff = "10.0.0"
 reqwest = { version = "0.11", features = ["blocking", "json"] } # TODO: Remove blocking when async is ready
 semver = { version = "1.0.4", features = ["serde"] }
 slug = "0.1.1"
@@ -49,11 +49,11 @@ schemamama = "0.3"
 schemamama_postgres = "0.3"
 systemstat = "0.1.4"
 prometheus = { version = "0.13.0", default-features = false }
-rustwide = "0.14.0"
+rustwide = "0.15.0"
 mime_guess = "2"
 dotenv = "0.15"
 zstd = "0.11.0"
-git2 = { version = "0.13.6", default-features = false }
+git2 = { version = "0.14.4", default-features = false }
 path-slash = "0.1.3"
 once_cell = { version = "1.4.0", features = ["parking_lot"] }
 base64 = "0.13"
@@ -116,7 +116,7 @@ test-case = "2.0.0"
 
 [build-dependencies]
 time = "0.3"
-git2 = { version = "0.13", default-features = false }
+git2 = { version = "0.14", default-features = false }
 sass-rs = "0.2.2"
 string_cache_codegen = "0.5.1"
 walkdir = "2"

Cargo.toml

@@ -452,7 +452,13 @@ impl DatabaseSubcommand {
             }
             Self::Delete {
                 command: DeleteSubcommand::Crate { name },
-            } => db::delete_crate(&ctx, &name).context("failed to delete the crate")?,
+            } => db::delete_crate(
+                &mut *ctx.pool()?.get()?,
+                &*ctx.storage()?,
+                &*ctx.config()?,
+                &name,
+            )
+            .context("failed to delete the crate")?,
             Self::Blacklist { command } => command.handle_args(ctx)?,
 
             #[cfg(feature = "consistency_check")]
@@ -570,6 +576,7 @@ impl Context for BinContext {
             self.pool()?,
             self.metrics()?,
             self.config()?,
+            self.storage()?,
         );
         fn storage(self) -> Storage = Storage::new(
             self.pool()?,

src/bin/cratesfyi.rs

@@ -1,11 +1,12 @@
-use crate::db::Pool;
+use crate::db::{delete_crate, Pool};
 use crate::docbuilder::PackageKind;
 use crate::error::Result;
+use crate::storage::Storage;
 use crate::utils::{get_crate_priority, report_error};
 use crate::{Config, Index, Metrics, RustwideBuilder};
 use anyhow::Context;
 
-use crates_index_diff::ChangeKind;
+use crates_index_diff::Change;
 use log::debug;
 
 use std::fs;
@@ -25,18 +26,25 @@ pub(crate) struct QueuedCrate {
 #[derive(Debug)]
 pub struct BuildQueue {
     config: Arc<Config>,
+    storage: Arc<Storage>,
     pub(crate) db: Pool,
     metrics: Arc<Metrics>,
     max_attempts: i32,
 }
 
 impl BuildQueue {
-    pub fn new(db: Pool, metrics: Arc<Metrics>, config: Arc<Config>) -> Self {
+    pub fn new(
+        db: Pool,
+        metrics: Arc<Metrics>,
+        config: Arc<Config>,
+        storage: Arc<Storage>,
+    ) -> Self {
         BuildQueue {
             max_attempts: config.build_attempts.into(),
             config,
             db,
             metrics,
+            storage,
         }
     }
 
@@ -194,9 +202,9 @@ impl BuildQueue {
         // I believe this will fix ordering of queue if we get more than one crate from changes
         changes.reverse();
 
-        for krate in &changes {
-            match krate.kind {
-                ChangeKind::Yanked => {
+        for change in &changes {
+            match change {
+                Change::Yanked(release) => {
                     let res = conn
                         .execute(
                             "
@@ -207,43 +215,51 @@ impl BuildQueue {
                                 AND name = $1
                                 AND version = $2
                             ",
-                            &[&krate.name, &krate.version],
+                            &[&release.name, &release.version],
                         )
                         .with_context(|| {
                             format!(
                                 "error while setting {}-{} to yanked",
-                                krate.name, krate.version
+                                release.name, release.version
                             )
                         });
                     match res {
-                        Ok(_) => debug!("{}-{} yanked", krate.name, krate.version),
+                        Ok(_) => debug!("{}-{} yanked", release.name, release.version),
                         Err(err) => report_error(&err),
                     }
                 }
 
-                ChangeKind::Added => {
-                    let priority = get_crate_priority(&mut conn, &krate.name)?;
+                Change::Added(release) => {
+                    let priority = get_crate_priority(&mut conn, &release.name)?;
 
                     match self
                         .add_crate(
-                            &krate.name,
-                            &krate.version,
+                            &release.name,
+                            &release.version,
                             priority,
                             index.repository_url(),
                         )
                         .with_context(|| {
                             format!(
                                 "failed adding {}-{} into build queue",
-                                krate.name, krate.version
+                                release.name, release.version
                             )
                         }) {
                         Ok(()) => {
-                            debug!("{}-{} added into build queue", krate.name, krate.version);
+                            debug!(
+                                "{}-{} added into build queue",
+                                release.name, release.version
+                            );
                             crates_added += 1;
                         }
                         Err(err) => report_error(&err),
                     }
                 }
+
+                Change::Deleted(krate) => {
+                    delete_crate(&mut conn, &self.storage, &self.config, krate)
+                        .with_context(|| format!("failed to delete crate {}", krate))?;
+                }
             }
         }
 

src/build_queue.rs

@@ -1,6 +1,6 @@
 use crate::error::Result;
-use crate::storage::{rustdoc_archive_path, source_archive_path};
-use crate::Context;
+use crate::storage::{rustdoc_archive_path, source_archive_path, Storage};
+use crate::{Config, Context};
 use anyhow::Context as _;
 use postgres::Client;
 use std::fs;
@@ -16,8 +16,12 @@ enum CrateDeletionError {
     MissingCrate(String),
 }
 
-pub fn delete_crate(ctx: &dyn Context, name: &str) -> Result<()> {
-    let conn = &mut ctx.pool()?.get()?;
+pub fn delete_crate(
+    conn: &mut Client,
+    storage: &Storage,
+    config: &Config,
+    name: &str,
+) -> Result<()> {
     let crate_id = get_id(conn, name)?;
     let is_library = delete_crate_from_database(conn, name, crate_id)?;
     // #899
@@ -31,10 +35,10 @@ pub fn delete_crate(ctx: &dyn Context, name: &str) -> Result<()> {
         // delete the whole rustdoc/source folder for this crate.
         // it will include existing archives.
         let remote_folder = format!("{}/{}/", prefix, name);
-        ctx.storage()?.delete_prefix(&remote_folder)?;
+        storage.delete_prefix(&remote_folder)?;
 
         // remove existing local archive index files.
-        let local_index_folder = ctx.config()?.local_archive_cache_path.join(&remote_folder);
+        let local_index_folder = config.local_archive_cache_path.join(&remote_folder);
         if local_index_folder.exists() {
             fs::remove_dir_all(&local_index_folder).with_context(|| {
                 format!(
@@ -244,7 +248,7 @@ mod tests {
                 )?);
             }
 
-            delete_crate(env, "package-1")?;
+            delete_crate(&mut db.conn(), &env.storage(), &env.config(), "package-1")?;
 
             assert!(!crate_exists(&mut db.conn(), "package-1")?);
             assert!(crate_exists(&mut db.conn(), "package-2")?);

src/db/delete.rs

@@ -19,7 +19,7 @@ use postgres::Client;
 use rustwide::cmd::{Command, CommandError, SandboxBuilder, SandboxImage};
 use rustwide::logging::{self, LogStorage};
 use rustwide::toolchain::ToolchainError;
-use rustwide::{Build, Crate, Toolchain, Workspace, WorkspaceBuilder};
+use rustwide::{AlternativeRegistry, Build, Crate, Toolchain, Workspace, WorkspaceBuilder};
 use serde_json::Value;
 use std::collections::{HashMap, HashSet};
 use std::path::Path;
@@ -318,7 +318,9 @@ impl RustwideBuilder {
         let krate = match kind {
             PackageKind::Local(path) => Crate::local(path),
             PackageKind::CratesIo => Crate::crates_io(name, version),
-            PackageKind::Registry(registry) => Crate::registry(registry, name, version),
+            PackageKind::Registry(registry) => {
+                Crate::registry(AlternativeRegistry::new(registry), name, version)
+            }
         };
         krate.fetch(&self.workspace).map_err(FailureError::compat)?;
 

src/docbuilder/rustwide_builder.rs

@@ -188,6 +188,7 @@ impl TestEnvironment {
                     self.db().pool(),
                     self.metrics(),
                     self.config(),
+                    self.storage(),
                 ))
             })
             .clone()