diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20f544924..043f58f63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,10 +58,10 @@ jobs: psql "${CRATESFYI_DATABASE_URL}" - name: Build - run: cargo build --locked + run: cargo build --workspace --locked - name: Test - run: cargo test --locked + run: cargo test --workspace --locked - name: Clean up the database run: docker-compose down --volumes @@ -134,7 +134,7 @@ jobs: - uses: actions-rs/cargo@v1 with: command: clippy - args: --locked -- -D warnings + args: --workspace --locked -- -D warnings - name: Clean unused artifacts uses: actions-rs/cargo@v1 diff --git a/Cargo.lock b/Cargo.lock index 3f439d1d7..4f29a6a47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -359,6 +359,7 @@ dependencies = [ "crates-index 0.15.1 (registry+https://github.com/rust-lang/crates.io-index)", "crates-index-diff 7.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "criterion 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "docsrs-metadata 0.1.0", "dotenv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -625,6 +626,15 @@ name = "discard" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "docsrs-metadata" +version = "0.1.0" +dependencies = [ + "serde 1.0.110 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.20 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "dotenv" version = "0.15.0" diff --git a/Cargo.toml b/Cargo.toml index 7bef59428..2d552cf6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,8 @@ repository = "https://github.com/rust-lang/docs.rs" build = "build.rs" edition = "2018" +[workspace] + [dependencies] log = "0.4" regex = "1" @@ -24,6 +26,7 @@ r2d2_postgres = "0.16" # url@2 for other usecases url = { version = "2.1.1", features = ["serde"] } badge = { path = "src/web/badge" } +docsrs-metadata = { path = "metadata" } backtrace = "0.3" failure = { version = "0.1.3", features = ["backtrace"] } comrak = { version = "0.8", default-features = false } diff --git a/build.rs b/build.rs index 9a9fc9114..125976540 100644 --- a/build.rs +++ b/build.rs @@ -8,12 +8,6 @@ use std::{ }; fn main() { - // Set the host target - println!( - "cargo:rustc-env=CRATESFYI_HOST_TARGET={}", - env::var("TARGET").unwrap(), - ); - // Don't rerun anytime a single change is made println!("cargo:rerun-if-changed=templates/style/vendored.scss"); println!("cargo:rerun-if-changed=templates/style/base.scss"); diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index d6dd4dc22..ff95da17b 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -31,6 +31,7 @@ WORKDIR /build COPY benches benches COPY Cargo.lock Cargo.toml ./ COPY src/web/badge src/web/badge/ +COPY metadata metadata/ RUN echo "fn main() {}" > src/main.rs && \ echo "fn main() {}" > build.rs diff --git a/metadata/Cargo.toml b/metadata/Cargo.toml new file mode 100644 index 000000000..7a573ea4d --- /dev/null +++ b/metadata/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "docsrs-metadata" +version = "0.1.0" +authors = ["Joshua Nelson "] +edition = "2018" +license = "MIT" +repository = "https://github.com/rust-lang/docs.rs" +description = "Document crates the same way docs.rs would" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +path = "lib.rs" + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +toml = { version = "0.5", default-features = false } +thiserror = "1" diff --git a/metadata/build.rs b/metadata/build.rs new file mode 100644 index 000000000..9cc7fadbe --- /dev/null +++ b/metadata/build.rs @@ -0,0 +1,10 @@ +fn main() { + // Set the host target + println!( + "cargo:rustc-env=DOCS_RS_METADATA_HOST_TARGET={}", + std::env::var("TARGET").unwrap(), + ); + // This only needs to be rerun if the TARGET changed, in which case cargo reruns it anyway. + // See https://doc.rust-lang.org/cargo/reference/build-scripts.html#cargorerun-if-env-changedname + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/metadata/lib.rs b/metadata/lib.rs new file mode 100644 index 000000000..b6a14d6cb --- /dev/null +++ b/metadata/lib.rs @@ -0,0 +1,627 @@ +#![warn(missing_docs)] +// N.B. requires nightly rustdoc to document until intra-doc links are stabilized. +//! Collect information that allows you to build a crate the same way that docs.rs would. +//! +//! This library is intended for use in docs.rs and crater, but might be helpful to others. +//! See for more information about the flags that can be set. +//! +//! This crate can only be used with nightly versions of `cargo` and `rustdoc`, because it +//! will always have the flag `-Z unstable-options`. +//! +//! Here is an example use of the crate: +//! +//! ``` +//! # fn main() -> Result<(), Box> { +//! use std::process::Command; +//! use docsrs_metadata::Metadata; +//! +//! // First, we need to parse Cargo.toml. +//! let source_root = env!("CARGO_MANIFEST_DIR"); +//! let metadata = Metadata::from_crate_root(&source_root)?; +//! +//! // Next, learn what arguments we need to pass to `cargo`. +//! let targets = metadata.targets(); +//! let mut cargo_args = metadata.cargo_args(); +//! cargo_args.push(targets.default_target.into()); +//! +//! // Now, set up the `Command` +//! let mut cmd = Command::new("cargo"); +//! cmd.args(cargo_args); +//! for (key, value) in metadata.environment_variables() { +//! cmd.env(key, value); +//! } +//! +//! // Finally, run `cargo doc` on the directory. +//! let result = cmd.output()?; +//! # Ok(()) +//! # } +//! ``` + +use std::collections::{HashMap, HashSet}; +use std::io; +use std::path::Path; + +use serde::Deserialize; +use thiserror::Error; +use toml::Value; + +/// The target that `metadata` is being built for. +/// +/// This is directly passed on from the Cargo [`TARGET`] variable. +/// +/// [`TARGET`]: https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-build-scripts +pub const HOST_TARGET: &str = env!("DOCS_RS_METADATA_HOST_TARGET"); +/// The targets that are built if no `targets` section is specified. +/// +/// Currently, this is guaranteed to have only [tier one] targets. +/// However, it may not contain all tier one targets. +/// +/// [tier one]: https://doc.rust-lang.org/nightly/rustc/platform-support.html#tier-1 +pub const DEFAULT_TARGETS: &[&str] = &[ + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", +]; + +/// The possible errors for [`Metadata::from_crate_root`]. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum MetadataError { + /// The error returned when the manifest could not be read. + #[error("failed to read manifest from disk")] + IO(#[from] io::Error), + /// The error returned when the manifest could not be parsed. + #[error("failed to parse manifest")] + Parse(#[from] toml::de::Error), +} + +/// Metadata to set for custom builds. +/// +/// This metadata is read from `[package.metadata.docs.rs]` table in `Cargo.toml`. +/// +/// An example metadata: +/// +/// ```text +/// [package] +/// name = "test" +/// +/// [package.metadata.docs.rs] +/// features = [ "feature1", "feature2" ] +/// all-features = true +/// no-default-features = true +/// default-target = "x86_64-unknown-linux-gnu" +/// targets = [ "x86_64-apple-darwin", "x86_64-pc-windows-msvc" ] +/// rustc-args = [ "--example-rustc-arg" ] +/// rustdoc-args = [ "--example-rustdoc-arg" ] +/// ``` +/// +/// You can define one or more fields in your `Cargo.toml`. +#[derive(Default, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct Metadata { + /// List of features to pass on to `cargo`. + /// + /// By default, docs.rs will only build default features. + features: Option>, + + /// Whether to pass `--all-features` to `cargo`. + #[serde(default)] + all_features: bool, + + /// Whether to pass `--no-default-features` to `cargo`. + // + /// By default, Docs.rs will build default features. + /// Set `no-default-fatures` to `true` if you want to build only certain features. + #[serde(default)] + no_default_features: bool, + + /// See [`BuildTargets`]. + default_target: Option, + targets: Option>, + + /// List of command line arguments for `rustc`. + rustc_args: Option>, + + /// List of command line arguments for `rustdoc`. + #[serde(default)] + rustdoc_args: Vec, +} + +/// The targets that should be built for a crate. +/// +/// The `default_target` is the target to be used as the home page for that crate. +/// +/// # See also +/// - [`Metadata::targets`] +pub struct BuildTargets<'a> { + /// The target that should be built by default. + /// + /// If `default_target` is unset and `targets` is non-empty, + /// the first element of `targets` will be used as the `default_target`. + /// Otherwise, this defaults to [`HOST_TARGET`]. + pub default_target: &'a str, + + /// Which targets should be built. + /// + /// If you want a crate to build only for specific targets, + /// set `targets` to the list of targets to build, in addition to `default-target`. + /// + /// If `targets` is not set, all [`DEFAULT_TARGETS`] will be built. + /// If `targets` is set to an empty array, only the default target will be built. + /// If `targets` is set to a non-empty array but `default_target` is not set, + /// the first element will be treated as the default. + pub other_targets: HashSet<&'a str>, +} + +impl Metadata { + /// Read the `Cargo.toml` from a source directory, then parse the build metadata. + /// + /// If both `Cargo.toml` and `Cargo.toml.orig` exist in the directory, + /// `Cargo.toml.orig` will take precedence. + /// + /// If you already have the path to a TOML file, use [`Metadata::from_manifest`] instead. + pub fn from_crate_root>(source_dir: P) -> Result { + let source_dir = source_dir.as_ref(); + for &c in &["Cargo.toml.orig", "Cargo.toml"] { + let manifest_path = source_dir.join(c); + if manifest_path.exists() { + return Metadata::from_manifest(manifest_path); + } + } + + Err(io::Error::new(io::ErrorKind::NotFound, "no Cargo.toml").into()) + } + + /// Read the given file into a string, then parse the build metadata. + /// + /// If you already have the TOML as a string, use [`from_str`] instead. + /// If you just want the default settings, use [`Metadata::default()`][Default::default]. + /// + /// [`from_str`]: std::str::FromStr + pub fn from_manifest>(path: P) -> Result { + use std::{fs, str::FromStr}; + let buf = fs::read_to_string(path)?; + Metadata::from_str(&buf).map_err(Into::into) + } + + /// Return the targets that should be built. + /// + /// The `default_target` will never be one of the `other_targets`. + pub fn targets(&self) -> BuildTargets<'_> { + let default_target = self + .default_target + .as_deref() + // Use the first element of `targets` if `default_target` is unset and `targets` is non-empty + .or_else(|| { + self.targets + .as_ref() + .and_then(|targets| targets.iter().next().map(String::as_str)) + }) + .unwrap_or(HOST_TARGET); + + // Let people opt-in to only having specific targets + let mut targets: HashSet<_> = self + .targets + .as_ref() + .map(|targets| targets.iter().map(String::as_str).collect()) + .unwrap_or_else(|| DEFAULT_TARGETS.iter().copied().collect()); + + targets.remove(&default_target); + BuildTargets { + default_target, + other_targets: targets, + } + } + + /// Return the arguments that should be passed to `cargo`. + /// + // TODO: maybe it shouldn't? + /// This will always include `doc --lib --no-deps`. + /// This will never include `--target`. + /// + /// Note that this does not necessarily reproduce the HTML _output_ of docs.rs exactly. + /// For example, the links may point somewhere different than they would on docs.rs. + /// However, rustdoc will see exactly the same code as it would on docs.rs, even counting `cfg`s. + pub fn cargo_args(&self) -> Vec { + let mut cargo_args: Vec = vec!["doc".into(), "--lib".into(), "--no-deps".into()]; + + if let Some(features) = &self.features { + cargo_args.push("--features".into()); + cargo_args.push(features.join(" ")); + } + + if self.all_features { + cargo_args.push("--all-features".into()); + } + + if self.no_default_features { + cargo_args.push("--no-default-features".into()); + } + + cargo_args + } + + /// Return the environment variables that should be set when building this crate. + /// + /// This will always contain at least `RUSTDOCFLAGS="-Z unstable-options"`. + pub fn environment_variables(&self) -> HashMap<&'static str, String> { + let joined = |v: &Option>| v.as_ref().map(|args| args.join(" ")).unwrap_or_default(); + + let mut map = HashMap::new(); + map.insert("RUSTFLAGS", joined(&self.rustc_args)); + map.insert("RUSTDOCFLAGS", self.rustdoc_args.join(" ")); + // For docs.rs detection from build scripts: + // https://github.com/rust-lang/docs.rs/issues/147 + map.insert("DOCS_RS", "1".into()); + + map + } +} + +impl std::str::FromStr for Metadata { + type Err = toml::de::Error; + + /// Parse the given manifest as TOML. + fn from_str(manifest: &str) -> Result { + use toml::value::Table; + + let manifest = match manifest.parse::()? { + Value::Table(t) => Some(t), + _ => None, + }; + + fn table(mut manifest: Table, table_name: &str) -> Option { + match manifest.remove(table_name) { + Some(Value::Table(table)) => Some(table), + _ => None, + } + } + + let table = manifest + .and_then(|t| table(t, "package")) + .and_then(|t| table(t, "metadata")) + .and_then(|t| table(t, "docs")) + .and_then(|t| table(t, "rs")); + let mut metadata = if let Some(table) = table { + Value::Table(table).try_into()? + } else { + Metadata::default() + }; + + metadata.rustdoc_args.push("-Z".into()); + metadata.rustdoc_args.push("unstable-options".into()); + + Ok(metadata) + } +} + +#[cfg(test)] +mod test_parsing { + use super::*; + use std::str::FromStr; + + #[test] + fn test_cratesfyi_metadata() { + let manifest = r#" + [package] + name = "test" + + [package.metadata.docs.rs] + features = [ "feature1", "feature2" ] + all-features = true + no-default-features = true + default-target = "x86_64-unknown-linux-gnu" + targets = [ "x86_64-apple-darwin", "x86_64-pc-windows-msvc" ] + rustc-args = [ "--example-rustc-arg" ] + rustdoc-args = [ "--example-rustdoc-arg" ] + "#; + + let metadata = Metadata::from_str(manifest).unwrap(); + + assert!(metadata.features.is_some()); + assert!(metadata.all_features); + assert!(metadata.no_default_features); + assert!(metadata.default_target.is_some()); + + let features = metadata.features.unwrap(); + assert_eq!(features.len(), 2); + assert_eq!(features[0], "feature1".to_owned()); + assert_eq!(features[1], "feature2".to_owned()); + + assert_eq!( + metadata.default_target.unwrap(), + "x86_64-unknown-linux-gnu".to_owned() + ); + + let targets = metadata.targets.expect("should have explicit target"); + assert_eq!(targets.len(), 2); + assert_eq!(targets[0], "x86_64-apple-darwin"); + assert_eq!(targets[1], "x86_64-pc-windows-msvc"); + + let rustc_args = metadata.rustc_args.unwrap(); + assert_eq!(rustc_args.len(), 1); + assert_eq!(rustc_args[0], "--example-rustc-arg".to_owned()); + + let rustdoc_args = metadata.rustdoc_args; + assert_eq!(rustdoc_args.len(), 3); + assert_eq!(rustdoc_args[0], "--example-rustdoc-arg".to_owned()); + assert_eq!(rustdoc_args[1], "-Z".to_owned()); + assert_eq!(rustdoc_args[2], "unstable-options".to_owned()); + } + + #[test] + fn test_no_targets() { + // metadata section but no targets + let manifest = r#" + [package] + name = "test" + + [package.metadata.docs.rs] + features = [ "feature1", "feature2" ] + "#; + let metadata = Metadata::from_str(manifest).unwrap(); + assert!(metadata.targets.is_none()); + + // no package.metadata.docs.rs section + let metadata = Metadata::from_str( + r#" + [package] + name = "test" + "#, + ) + .unwrap(); + assert!(metadata.targets.is_none()); + + // targets explicitly set to empty array + let metadata = Metadata::from_str( + r#" + [package.metadata.docs.rs] + targets = [] + "#, + ) + .unwrap(); + assert!(metadata.targets.unwrap().is_empty()); + } +} + +#[cfg(test)] +mod test_targets { + use super::*; + + #[test] + fn test_select_targets() { + use super::BuildTargets; + + let mut metadata = Metadata::default(); + + // unchanged default_target, targets not specified + let BuildTargets { + default_target: default, + other_targets: tier_one, + } = metadata.targets(); + assert_eq!(default, HOST_TARGET); + + // should be equal to TARGETS \ {HOST_TARGET} + for actual in &tier_one { + assert!(DEFAULT_TARGETS.contains(actual)); + } + + for expected in DEFAULT_TARGETS { + if *expected == HOST_TARGET { + assert!(!tier_one.contains(&HOST_TARGET)); + } else { + assert!(tier_one.contains(expected)); + } + } + + // unchanged default_target, targets specified to be empty + metadata.targets = Some(Vec::new()); + + let BuildTargets { + default_target: default, + other_targets: others, + } = metadata.targets(); + + assert_eq!(default, HOST_TARGET); + assert!(others.is_empty()); + + // unchanged default_target, targets non-empty + metadata.targets = Some(vec![ + "i686-pc-windows-msvc".into(), + "i686-apple-darwin".into(), + ]); + + let BuildTargets { + default_target: default, + other_targets: others, + } = metadata.targets(); + + assert_eq!(default, "i686-pc-windows-msvc"); + assert_eq!(others.len(), 1); + assert!(others.contains(&"i686-apple-darwin")); + + // make sure that default_target is not built twice + metadata.targets = Some(vec![HOST_TARGET.into()]); + let BuildTargets { + default_target: default, + other_targets: others, + } = metadata.targets(); + + assert_eq!(default, HOST_TARGET); + assert!(others.is_empty()); + + // make sure that duplicates are removed + metadata.targets = Some(vec![ + "i686-pc-windows-msvc".into(), + "i686-pc-windows-msvc".into(), + ]); + + let BuildTargets { + default_target: default, + other_targets: others, + } = metadata.targets(); + + assert_eq!(default, "i686-pc-windows-msvc"); + assert!(others.is_empty()); + + // make sure that `default_target` always takes priority over `targets` + metadata.default_target = Some("i686-apple-darwin".into()); + let BuildTargets { + default_target: default, + other_targets: others, + } = metadata.targets(); + + assert_eq!(default, "i686-apple-darwin"); + assert_eq!(others.len(), 1); + assert!(others.contains(&"i686-pc-windows-msvc")); + + // make sure that `default_target` takes priority over `HOST_TARGET` + metadata.targets = Some(vec![]); + let BuildTargets { + default_target: default, + other_targets: others, + } = metadata.targets(); + + assert_eq!(default, "i686-apple-darwin"); + assert!(others.is_empty()); + + // and if `targets` is unset, it should still be set to `TARGETS` + metadata.targets = None; + let BuildTargets { + default_target: default, + other_targets: others, + } = metadata.targets(); + + assert_eq!(default, "i686-apple-darwin"); + let tier_one_targets_no_default = DEFAULT_TARGETS + .iter() + .filter(|&&t| t != "i686-apple-darwin") + .copied() + .collect(); + + assert_eq!(others, tier_one_targets_no_default); + } +} + +#[cfg(test)] +mod test_calculations { + use super::*; + + fn default_cargo_args() -> Vec { + vec!["doc".into(), "--lib".into(), "--no-deps".into()] + } + + #[test] + fn test_defaults() { + let metadata = Metadata::default(); + assert_eq!(metadata.cargo_args(), default_cargo_args()); + let env = metadata.environment_variables(); + assert_eq!(env.get("DOCS_RS").map(String::as_str), Some("1")); + assert_eq!(env.get("RUSTDOCFLAGS").map(String::as_str), Some("")); + assert_eq!(env.get("RUSTFLAGS").map(String::as_str), Some("")); + } + + #[test] + fn test_features() { + // all features + let metadata = Metadata { + all_features: true, + ..Metadata::default() + }; + let mut expected_args = default_cargo_args(); + expected_args.push("--all-features".into()); + assert_eq!(metadata.cargo_args(), expected_args); + + // no default features + let metadata = Metadata { + no_default_features: true, + ..Metadata::default() + }; + let mut expected_args = default_cargo_args(); + expected_args.push("--no-default-features".into()); + assert_eq!(metadata.cargo_args(), expected_args); + + // allow passing both even though it's nonsense; cargo will give an error anyway + let metadata = Metadata { + all_features: true, + no_default_features: true, + ..Metadata::default() + }; + let mut expected_args = default_cargo_args(); + expected_args.push("--all-features".into()); + expected_args.push("--no-default-features".into()); + assert_eq!(metadata.cargo_args(), expected_args); + + // explicit empty vec + let metadata = Metadata { + features: Some(vec![]), + ..Metadata::default() + }; + let mut expected_args = default_cargo_args(); + expected_args.push("--features".into()); + expected_args.push(String::new()); + assert_eq!(metadata.cargo_args(), expected_args); + + // one feature + let metadata = Metadata { + features: Some(vec!["some_feature".into()]), + ..Metadata::default() + }; + let mut expected_args = default_cargo_args(); + expected_args.push("--features".into()); + expected_args.push("some_feature".into()); + assert_eq!(metadata.cargo_args(), expected_args); + + // multiple features + let metadata = Metadata { + features: Some(vec!["feature1".into(), "feature2".into()]), + ..Metadata::default() + }; + let mut expected_args = default_cargo_args(); + expected_args.push("--features".into()); + expected_args.push("feature1 feature2".into()); + assert_eq!(metadata.cargo_args(), expected_args); + + // rustdocflags + let metadata = Metadata { + rustdoc_args: vec![ + "-Z".into(), + "unstable-options".into(), + "--static-root-path".into(), + "/".into(), + "--cap-lints".into(), + "warn".into(), + ], + ..Metadata::default() + }; + assert_eq!( + metadata + .environment_variables() + .get("RUSTDOCFLAGS") + .map(String::as_str), + Some("-Z unstable-options --static-root-path / --cap-lints warn") + ); + + // rustdocflags + let metadata = Metadata { + rustc_args: Some(vec![ + "-Z".into(), + "unstable-options".into(), + "--static-root-path".into(), + "/".into(), + "--cap-lints".into(), + "warn".into(), + ]), + ..Metadata::default() + }; + assert_eq!( + metadata + .environment_variables() + .get("RUSTFLAGS") + .map(String::as_str), + Some("-Z unstable-options --static-root-path / --cap-lints warn") + ); + } +} diff --git a/src/docbuilder/metadata.rs b/src/docbuilder/metadata.rs deleted file mode 100644 index 13a753082..000000000 --- a/src/docbuilder/metadata.rs +++ /dev/null @@ -1,411 +0,0 @@ -use crate::error::Result; -use failure::err_msg; -use std::collections::HashSet; -use std::path::Path; -use toml::{map::Map, Value}; - -/// Metadata for custom builds -/// -/// You can customize docs.rs builds by defining `[package.metadata.docs.rs]` table in your -/// crates' `Cargo.toml`. -/// -/// An example metadata: -/// -/// ```text -/// [package] -/// name = "test" -/// -/// [package.metadata.docs.rs] -/// features = [ "feature1", "feature2" ] -/// all-features = true -/// no-default-features = true -/// default-target = "x86_64-unknown-linux-gnu" -/// targets = [ "x86_64-apple-darwin", "x86_64-pc-windows-msvc" ] -/// rustc-args = [ "--example-rustc-arg" ] -/// rustdoc-args = [ "--example-rustdoc-arg" ] -/// ``` -/// -/// You can define one or more fields in your `Cargo.toml`. -pub struct Metadata { - /// List of features docs.rs will build. - /// - /// By default, docs.rs will only build default features. - pub features: Option>, - - /// Set `all-features` to true if you want docs.rs to build all features for your crate - pub all_features: bool, - - /// Docs.rs will always build default features. - /// - /// Set `no-default-fatures` to `false` if you want to build only certain features. - pub no_default_features: bool, - - /// docs.rs runs on `x86_64-unknown-linux-gnu`, which is the default target for documentation by default. - /// - /// You can change the default target by setting this. - /// - /// If `default_target` is unset and `targets` is non-empty, - /// the first element of `targets` will be used as the `default_target`. - pub default_target: Option, - - /// If you want a crate to build only for specific targets, - /// set `targets` to the list of targets to build, in addition to `default-target`. - /// - /// If you do not set `targets`, all of the tier 1 supported targets will be built. - /// If you set `targets` to an empty array, only the default target will be built. - /// If you set `targets` to a non-empty array but do not set `default_target`, - /// the first element will be treated as the default. - pub targets: Option>, - - /// List of command line arguments for `rustc`. - pub rustc_args: Option>, - - /// List of command line arguments for `rustdoc`. - pub rustdoc_args: Option>, -} - -/// The targets that should be built for a crate. -/// -/// The `default_target` is the target to be used as the home page for that crate. -/// -/// # See also -/// - [`Metadata::targets`](struct.Metadata.html#method.targets) -pub(super) struct BuildTargets<'a> { - pub(super) default_target: &'a str, - pub(super) other_targets: HashSet<&'a str>, -} - -impl Metadata { - pub(crate) fn from_source_dir(source_dir: &Path) -> Result { - for &c in &["Cargo.toml.orig", "Cargo.toml"] { - let manifest_path = source_dir.join(c); - if manifest_path.exists() { - return Ok(Metadata::from_manifest(manifest_path)); - } - } - - Err(err_msg("Manifest not found")) - } - - fn from_manifest>(path: P) -> Metadata { - use std::{fs::File, io::Read}; - - let mut file = if let Ok(file) = File::open(path) { - file - } else { - return Metadata::default(); - }; - - let mut meta = String::new(); - if file.read_to_string(&mut meta).is_err() { - return Metadata::default(); - } - - Metadata::from_str(&meta) - } - - // This is similar to Default trait but it's private - fn default() -> Metadata { - Metadata { - features: None, - all_features: false, - no_default_features: false, - default_target: None, - rustc_args: None, - rustdoc_args: None, - targets: None, - } - } - - fn from_str(manifest: &str) -> Metadata { - let mut metadata = Metadata::default(); - - let manifest = if let Ok(manifest) = manifest.parse::() { - manifest - } else { - return metadata; - }; - - fn fetch_manifest_tables<'a>(manifest: &'a Value) -> Option<&'a Map> { - manifest - .get("package")? - .as_table()? - .get("metadata")? - .as_table()? - .get("docs")? - .as_table()? - .get("rs")? - .as_table() - } - - if let Some(table) = fetch_manifest_tables(&manifest) { - let collect_into_array = - |f: &Vec| f.iter().map(|v| v.as_str().map(|v| v.to_owned())).collect(); - - metadata.features = table - .get("features") - .and_then(|f| f.as_array()) - .and_then(collect_into_array); - - metadata.no_default_features = table - .get("no-default-features") - .and_then(|v| v.as_bool()) - .unwrap_or(metadata.no_default_features); - - metadata.all_features = table - .get("all-features") - .and_then(|v| v.as_bool()) - .unwrap_or(metadata.all_features); - - metadata.default_target = table - .get("default-target") - .and_then(|v| v.as_str()) - .map(|v| v.to_owned()); - - metadata.targets = table - .get("targets") - .and_then(|f| f.as_array()) - .and_then(collect_into_array); - - metadata.rustc_args = table - .get("rustc-args") - .and_then(|f| f.as_array()) - .and_then(collect_into_array); - - metadata.rustdoc_args = table - .get("rustdoc-args") - .and_then(|f| f.as_array()) - .and_then(collect_into_array); - } - - metadata - } - - pub(super) fn targets(&self) -> BuildTargets<'_> { - use super::rustwide_builder::{HOST_TARGET, TARGETS}; - - let default_target = self - .default_target - .as_deref() - // Use the first element of `targets` if `default_target` is unset and `targets` is non-empty - .or_else(|| { - self.targets - .as_ref() - .and_then(|targets| targets.iter().next().map(String::as_str)) - }) - .unwrap_or(HOST_TARGET); - - // Let people opt-in to only having specific targets - let mut targets: HashSet<_> = self - .targets - .as_ref() - .map(|targets| targets.iter().map(String::as_str).collect()) - .unwrap_or_else(|| TARGETS.iter().copied().collect()); - - targets.remove(&default_target); - BuildTargets { - default_target, - other_targets: targets, - } - } -} - -#[cfg(test)] -mod test { - use super::Metadata; - - #[test] - fn test_cratesfyi_metadata() { - crate::test::init_logger(); - let manifest = r#" - [package] - name = "test" - - [package.metadata.docs.rs] - features = [ "feature1", "feature2" ] - all-features = true - no-default-features = true - default-target = "x86_64-unknown-linux-gnu" - targets = [ "x86_64-apple-darwin", "x86_64-pc-windows-msvc" ] - rustc-args = [ "--example-rustc-arg" ] - rustdoc-args = [ "--example-rustdoc-arg" ] - "#; - - let metadata = Metadata::from_str(manifest); - - assert!(metadata.features.is_some()); - assert!(metadata.all_features); - assert!(metadata.no_default_features); - assert!(metadata.default_target.is_some()); - assert!(metadata.rustdoc_args.is_some()); - - let features = metadata.features.unwrap(); - assert_eq!(features.len(), 2); - assert_eq!(features[0], "feature1".to_owned()); - assert_eq!(features[1], "feature2".to_owned()); - - assert_eq!( - metadata.default_target.unwrap(), - "x86_64-unknown-linux-gnu".to_owned() - ); - - let targets = metadata.targets.expect("should have explicit target"); - assert_eq!(targets.len(), 2); - assert_eq!(targets[0], "x86_64-apple-darwin"); - assert_eq!(targets[1], "x86_64-pc-windows-msvc"); - - let rustc_args = metadata.rustc_args.unwrap(); - assert_eq!(rustc_args.len(), 1); - assert_eq!(rustc_args[0], "--example-rustc-arg".to_owned()); - - let rustdoc_args = metadata.rustdoc_args.unwrap(); - assert_eq!(rustdoc_args.len(), 1); - assert_eq!(rustdoc_args[0], "--example-rustdoc-arg".to_owned()); - } - - #[test] - fn test_no_targets() { - // metadata section but no targets - let manifest = r#" - [package] - name = "test" - - [package.metadata.docs.rs] - features = [ "feature1", "feature2" ] - "#; - let metadata = Metadata::from_str(manifest); - assert!(metadata.targets.is_none()); - - // no package.metadata.docs.rs section - let metadata = Metadata::from_str( - r#" - [package] - name = "test" - "#, - ); - assert!(metadata.targets.is_none()); - - // targets explicitly set to empty array - let metadata = Metadata::from_str( - r#" - [package.metadata.docs.rs] - targets = [] - "#, - ); - assert!(metadata.targets.unwrap().is_empty()); - } - #[test] - fn test_select_targets() { - use super::BuildTargets; - use crate::docbuilder::rustwide_builder::{HOST_TARGET, TARGETS}; - - let mut metadata = Metadata::default(); - - // unchanged default_target, targets not specified - let BuildTargets { - default_target: default, - other_targets: tier_one, - } = metadata.targets(); - assert_eq!(default, HOST_TARGET); - - // should be equal to TARGETS \ {HOST_TARGET} - for actual in &tier_one { - assert!(TARGETS.contains(actual)); - } - - for expected in TARGETS { - if *expected == HOST_TARGET { - assert!(!tier_one.contains(&HOST_TARGET)); - } else { - assert!(tier_one.contains(expected)); - } - } - - // unchanged default_target, targets specified to be empty - metadata.targets = Some(Vec::new()); - - let BuildTargets { - default_target: default, - other_targets: others, - } = metadata.targets(); - - assert_eq!(default, HOST_TARGET); - assert!(others.is_empty()); - - // unchanged default_target, targets non-empty - metadata.targets = Some(vec![ - "i686-pc-windows-msvc".into(), - "i686-apple-darwin".into(), - ]); - - let BuildTargets { - default_target: default, - other_targets: others, - } = metadata.targets(); - - assert_eq!(default, "i686-pc-windows-msvc"); - assert_eq!(others.len(), 1); - assert!(others.contains(&"i686-apple-darwin")); - - // make sure that default_target is not built twice - metadata.targets = Some(vec![HOST_TARGET.into()]); - let BuildTargets { - default_target: default, - other_targets: others, - } = metadata.targets(); - - assert_eq!(default, HOST_TARGET); - assert!(others.is_empty()); - - // make sure that duplicates are removed - metadata.targets = Some(vec![ - "i686-pc-windows-msvc".into(), - "i686-pc-windows-msvc".into(), - ]); - - let BuildTargets { - default_target: default, - other_targets: others, - } = metadata.targets(); - - assert_eq!(default, "i686-pc-windows-msvc"); - assert!(others.is_empty()); - - // make sure that `default_target` always takes priority over `targets` - metadata.default_target = Some("i686-apple-darwin".into()); - let BuildTargets { - default_target: default, - other_targets: others, - } = metadata.targets(); - - assert_eq!(default, "i686-apple-darwin"); - assert_eq!(others.len(), 1); - assert!(others.contains(&"i686-pc-windows-msvc")); - - // make sure that `default_target` takes priority over `HOST_TARGET` - metadata.targets = Some(vec![]); - let BuildTargets { - default_target: default, - other_targets: others, - } = metadata.targets(); - - assert_eq!(default, "i686-apple-darwin"); - assert!(others.is_empty()); - - // and if `targets` is unset, it should still be set to `TARGETS` - metadata.targets = None; - let BuildTargets { - default_target: default, - other_targets: others, - } = metadata.targets(); - - assert_eq!(default, "i686-apple-darwin"); - let tier_one_targets_no_default = TARGETS - .iter() - .filter(|&&t| t != "i686-apple-darwin") - .copied() - .collect(); - - assert_eq!(others, tier_one_targets_no_default); - } -} diff --git a/src/docbuilder/mod.rs b/src/docbuilder/mod.rs index fb1277be0..220ad062e 100644 --- a/src/docbuilder/mod.rs +++ b/src/docbuilder/mod.rs @@ -1,12 +1,10 @@ mod crates; mod limits; -mod metadata; pub(crate) mod options; mod queue; mod rustwide_builder; pub(crate) use self::limits::Limits; -pub(self) use self::metadata::Metadata; pub use self::rustwide_builder::RustwideBuilder; pub(crate) use self::rustwide_builder::{BuildResult, DocCoverage}; diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 474d37675..d9b4c8942 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -1,5 +1,4 @@ use super::DocBuilder; -use super::Metadata; use crate::db::blacklist::is_blacklisted; use crate::db::file::add_path_into_database; use crate::db::{ @@ -12,6 +11,7 @@ use crate::index::api::ReleaseData; use crate::storage::CompressionAlgorithms; use crate::utils::{copy_doc_dir, parse_rustc_version, CargoMetadata}; use crate::{Metrics, Storage}; +use docsrs_metadata::{Metadata, DEFAULT_TARGETS, HOST_TARGET}; use failure::ResultExt; use log::{debug, info, warn, LevelFilter}; use rustwide::cmd::{Command, SandboxBuilder}; @@ -26,18 +26,6 @@ use std::sync::Arc; const USER_AGENT: &str = "docs.rs builder (https://github.com/rust-lang/docs.rs)"; const DEFAULT_RUSTWIDE_WORKSPACE: &str = ".rustwide"; - -// It is crucial that this be the same as the host that `docs.rs` is being run on. -// Other values may cause strange and hard-to-debug errors. -pub(super) const HOST_TARGET: &str = env!("CRATESFYI_HOST_TARGET"); // Set in build.rs -pub(super) const TARGETS: &[&str] = &[ - "i686-pc-windows-msvc", - "i686-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-msvc", - "x86_64-unknown-linux-gnu", -]; - const ESSENTIAL_FILES_VERSIONED: &[&str] = &[ "brush.svg", "wheel.svg", @@ -133,7 +121,7 @@ impl RustwideBuilder { // Ignore errors if detection fails. let old_version = self.detect_rustc_version().ok(); - let mut targets_to_install = TARGETS + let mut targets_to_install = DEFAULT_TARGETS .iter() .map(|&t| t.to_string()) // &str has a specialized ToString impl, while &&str goes through Display .collect::>(); @@ -227,7 +215,7 @@ impl RustwideBuilder { build_dir .build(&self.toolchain, &krate, self.prepare_sandbox(&limits)) .run(|build| { - let metadata = Metadata::from_source_dir(&build.host_source_dir())?; + let metadata = Metadata::from_crate_root(&build.host_source_dir())?; let res = self.execute_build(HOST_TARGET, true, build, &limits, &metadata)?; if !res.result.successful { @@ -349,11 +337,11 @@ impl RustwideBuilder { let res = build_dir .build(&self.toolchain, &krate, self.prepare_sandbox(&limits)) .run(|build| { - use crate::docbuilder::metadata::BuildTargets; + use docsrs_metadata::BuildTargets; let mut has_docs = false; let mut successful_targets = Vec::new(); - let metadata = Metadata::from_source_dir(&build.host_source_dir())?; + let metadata = Metadata::from_crate_root(&build.host_source_dir())?; let BuildTargets { default_target, other_targets, @@ -599,70 +587,37 @@ impl RustwideBuilder { limits: &Limits, rustdoc_flags_extras: Vec, ) -> Result> { - let mut cargo_args = vec!["doc", "--lib", "--no-deps"]; - if target != HOST_TARGET { - // If the explicit target is not a tier one target, we need to install it. - if !TARGETS.contains(&target) { - // This is a no-op if the target is already installed. - self.toolchain.add_target(&self.workspace, target)?; - } - cargo_args.push("--target"); - cargo_args.push(target); - }; - - let tmp; - if let Some(cpu_limit) = self.cpu_limit { - tmp = format!("-j{}", cpu_limit); - cargo_args.push(&tmp); - } - - let tmp; - if let Some(features) = &metadata.features { - cargo_args.push("--features"); - tmp = features.join(" "); - cargo_args.push(&tmp); - } - if metadata.all_features { - cargo_args.push("--all-features"); - } - if metadata.no_default_features { - cargo_args.push("--no-default-features"); + // If the explicit target is not a tier one target, we need to install it. + if !docsrs_metadata::DEFAULT_TARGETS.contains(&target) { + // This is a no-op if the target is already installed. + self.toolchain.add_target(&self.workspace, target)?; } - let mut rustdoc_flags = vec![ - "-Z".to_string(), - "unstable-options".to_string(), - "--static-root-path".to_string(), - "/".to_string(), - "--cap-lints".to_string(), - "warn".to_string(), - ]; + let mut cargo_args = metadata.cargo_args(); - if let Some(package_rustdoc_args) = &metadata.rustdoc_args { - rustdoc_flags.append(&mut package_rustdoc_args.clone()); + // Add docs.rs specific arguments + if let Some(cpu_limit) = self.cpu_limit { + cargo_args.push(format!("-j{}", cpu_limit)); } + if target != HOST_TARGET { + cargo_args.push("--target".into()); + cargo_args.push(target.into()); + }; - rustdoc_flags.extend(rustdoc_flags_extras); + let mut env_vars = metadata.environment_variables(); + let rustdoc_flags = env_vars.entry("RUSTDOCFLAGS").or_default(); + rustdoc_flags.push_str(" --static-root-path / --cap-lints warn "); + rustdoc_flags.push_str(&rustdoc_flags_extras.join(" ")); - let command = build + let mut command = build .cargo() .timeout(Some(limits.timeout())) - .no_output_timeout(None) - .env( - "RUSTFLAGS", - metadata - .rustc_args - .as_ref() - .map(|args| args.join(" ")) - .unwrap_or_default(), - ) - .env("RUSTDOCFLAGS", rustdoc_flags.join(" ")) - // For docs.rs detection from build script: - // https://github.com/rust-lang/docs.rs/issues/147 - .env("DOCS_RS", "1") - .args(&cargo_args); - - Ok(command) + .no_output_timeout(None); + for (key, val) in env_vars { + command = command.env(key, val); + } + + Ok(command.args(&cargo_args)) } fn copy_docs(