Allow to specify a key for private registries

Author: l4l

Cargo.lock

@@ -27,7 +27,7 @@ log = "0.4"
 regex = "1"
 structopt = "0.3"
 crates-index = { version = "0.15.1", optional = true }
-crates-index-diff = "7.1.1"
+crates-index-diff = "8.0.0"
 reqwest = { version = "0.11", features = ["blocking", "json"] } # TODO: Remove blocking when async is ready
 semver = { version = "0.9", features = ["serde"] }
 slug = "=0.1.1"
@@ -48,7 +48,7 @@ schemamama = "0.3"
 schemamama_postgres = "0.3"
 systemstat = "0.1.4"
 prometheus = { version = "0.10.0", default-features = false }
-rustwide = "0.13"
+rustwide = { git = "https://github.com/l4l/rustwide.git", branch = "registry-key" }
 mime_guess = "2"
 dotenv = "0.15"
 zstd = "0.5"

Cargo.toml

@@ -6,6 +6,7 @@ export DOCSRS_PREFIX=/opt/docsrs/prefix
 export DOCSRS_DOCKER=true
 export DOCSRS_LOG=${DOCSRS_LOG-"docs-rs,rustwide=info"}
 export PATH="$PATH:/build/target/release"
+export REGISTRY_URL=${REGISTRY_URL:-https://github.com/rust-lang/crates.io-index}
 
 # Try migrating the database multiple times if it fails
 # This avoids the docker container crashing the first time it's started with
@@ -28,7 +29,7 @@ done
 set -e
 
 if ! [ -d "${DOCSRS_PREFIX}/crates.io-index/.git" ]; then
-    git clone https://github.com/rust-lang/crates.io-index "${DOCSRS_PREFIX}/crates.io-index"
+    git clone ${REGISTRY_URL} "${DOCSRS_PREFIX}/crates.io-index"
     # Prevent new crates built before the container creation to be built
     git --git-dir="$DOCSRS_PREFIX/crates.io-index/.git" branch crates-index-diff_last-seen
 fi

dockerfiles/entrypoint.sh

@@ -334,7 +334,9 @@ impl BuildSubcommand {
                         .build_local_package(&path)
                         .context("Building documentation failed")?;
                 } else {
-                    let registry_url = ctx.config()?.registry_url.clone();
+                    let config = ctx.config()?;
+                    let registry_url = config.registry_url.clone();
+                    let registry_key = config.registry_key.clone();
                     builder
                         .build_package(
                             &crate_name
@@ -343,7 +345,10 @@ impl BuildSubcommand {
                                 .with_context(|| anyhow!("must specify version if not local"))?,
                             registry_url
                                 .as_ref()
-                                .map(|s| PackageKind::Registry(s.as_str()))
+                                .map(|s| PackageKind::Registry {
+                                    url: s.as_str(),
+                                    key: registry_key,
+                                })
                                 .unwrap_or(PackageKind::CratesIo),
                         )
                         .context("Building documentation failed")?;
@@ -598,7 +603,7 @@ impl Context for BinContext {
             let config = self.config()?;
             let path = config.registry_index_path.clone();
             if let Some(registry_url) = config.registry_url.clone() {
-                Index::from_url(path, registry_url)
+                Index::from_url(path, registry_url, config.registry_key.clone())
             } else {
                 Index::new(path)
             }?

src/bin/cratesfyi.rs

@@ -257,7 +257,10 @@ impl BuildQueue {
             let kind = krate
                 .registry
                 .as_ref()
-                .map(|r| PackageKind::Registry(r.as_str()))
+                .map(|r| PackageKind::Registry {
+                    url: r.as_str(),
+                    key: self.config.registry_key.clone(),
+                })
                 .unwrap_or(PackageKind::CratesIo);
 
             if let Err(err) = builder

src/build_queue.rs

@@ -11,6 +11,7 @@ pub struct Config {
     pub prefix: PathBuf,
     pub registry_index_path: PathBuf,
     pub registry_url: Option<String>,
+    pub registry_key: Option<String>,
 
     // Database connection params
     pub(crate) database_url: String,
@@ -98,6 +99,15 @@ impl Config {
 
             registry_index_path: env("REGISTRY_INDEX_PATH", prefix.join("crates.io-index"))?,
             registry_url: maybe_env("REGISTRY_URL")?,
+            registry_key: maybe_env::<String>("REGISTRY_KEY").and_then(|key| {
+                Ok(if let Some(key) = key {
+                    Some(key)
+                } else {
+                    maybe_env::<String>("REGISTRY_KEY_PATH")?
+                        .map(std::fs::read_to_string)
+                        .transpose()?
+                })
+            })?,
             prefix: prefix.clone(),
 
             database_url: require_env("DOCSRS_DATABASE_URL")?,

src/config.rs

@@ -19,7 +19,7 @@ use postgres::Client;
 use rustwide::cmd::{Command, CommandError, SandboxBuilder, SandboxImage};
 use rustwide::logging::{self, LogStorage};
 use rustwide::toolchain::ToolchainError;
-use rustwide::{Build, Crate, Toolchain, Workspace, WorkspaceBuilder};
+use rustwide::{AlternativeRegistry, Build, Crate, Toolchain, Workspace, WorkspaceBuilder};
 use serde_json::Value;
 use std::collections::{HashMap, HashSet};
 use std::path::Path;
@@ -32,7 +32,7 @@ const DUMMY_CRATE_VERSION: &str = "1.0.0";
 pub enum PackageKind<'a> {
     Local(&'a Path),
     CratesIo,
-    Registry(&'a str),
+    Registry { url: &'a str, key: Option<String> },
 }
 
 pub struct RustwideBuilder {
@@ -242,7 +242,10 @@ impl RustwideBuilder {
                 let registry_url = self.config.registry_url.clone();
                 let package_kind = registry_url
                     .as_ref()
-                    .map(|r| PackageKind::Registry(r.as_str()))
+                    .map(|r| PackageKind::Registry {
+                        url: r.as_str(),
+                        key: self.config.registry_key.clone(),
+                    })
                     .unwrap_or(PackageKind::CratesIo);
                 if let Err(err) = self.build_package(name, version, package_kind) {
                     warn!("failed to build package {} {}: {}", name, version, err);
@@ -309,7 +312,13 @@ impl RustwideBuilder {
         let krate = match kind {
             PackageKind::Local(path) => Crate::local(path),
             PackageKind::CratesIo => Crate::crates_io(name, version),
-            PackageKind::Registry(registry) => Crate::registry(registry, name, version),
+            PackageKind::Registry { url, key } => {
+                let mut registry = AlternativeRegistry::new(url);
+                if let Some(key) = key {
+                    registry.authenticate_with_ssh_key(key);
+                }
+                Crate::registry(registry, name, version)
+            }
         };
         krate.fetch(&self.workspace).map_err(FailureError::compat)?;
 

src/docbuilder/rustwide_builder.rs

@@ -15,6 +15,7 @@ pub struct Index {
     path: PathBuf,
     api: Api,
     repository_url: Option<String>,
+    registry_key: Option<String>,
 }
 
 #[derive(Debug, serde::Deserialize, Clone)]
@@ -39,12 +40,31 @@ fn load_config(repo: &git2::Repository) -> Result<IndexConfig> {
     Ok(config)
 }
 
+fn fetch_options(key: &str) -> git2::FetchOptions<'_> {
+    let mut fo = git2::FetchOptions::new();
+    fo.remote_callbacks({
+        let mut callbacks = git2::RemoteCallbacks::new();
+        callbacks.credentials(move |_url, username_from_url, _allowed_types| {
+            git2::Cred::ssh_key_from_memory(username_from_url.unwrap(), None, key, None)
+        });
+        callbacks
+    });
+    fo
+}
+
 impl Index {
-    pub fn from_url(path: PathBuf, repository_url: String) -> Result<Self> {
+    pub fn from_url(
+        path: PathBuf,
+        repository_url: String,
+        registry_key: Option<String>,
+    ) -> Result<Self> {
         let url = repository_url.clone();
         let diff = crates_index_diff::Index::from_path_or_cloned_with_options(
             &path,
-            crates_index_diff::CloneOptions { repository_url },
+            crates_index_diff::CloneOptions {
+                repository_url,
+                fetch_options: registry_key.as_ref().map(|s| fetch_options(s.as_str())),
+            },
         )
         .context("initialising registry index repository")?;
 
@@ -54,6 +74,7 @@ impl Index {
             path,
             api,
             repository_url: Some(url),
+            registry_key,
         })
     }
 
@@ -68,17 +89,23 @@ impl Index {
             path,
             api,
             repository_url: None,
+            registry_key: None,
         })
     }
 
     pub(crate) fn diff(&self) -> Result<crates_index_diff::Index> {
-        let options = self
-            .repository_url
-            .clone()
-            .map(|repository_url| crates_index_diff::CloneOptions { repository_url })
-            .unwrap_or_default();
-        let diff = crates_index_diff::Index::from_path_or_cloned_with_options(&self.path, options)
-            .context("re-opening registry index for diff")?;
+        let fetch_options = self.registry_key.as_deref().map(fetch_options);
+        let diff = crates_index_diff::Index::from_path_or_cloned_with_options(
+            &self.path,
+            self.repository_url
+                .clone()
+                .map(move |repository_url| crates_index_diff::CloneOptions {
+                    repository_url,
+                    fetch_options,
+                })
+                .unwrap_or_default(),
+        )
+        .context("re-opening registry index for diff")?;
         Ok(diff)
     }