From 7e36f87d5b06581451094db11319f38caa8a9205 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Wed, 19 Oct 2016 23:29:48 +0300 Subject: [PATCH 1/4] Add docs.rs metadata and custom builds This patch is adding `Metadata` type used in `cargo::ops::CompileOptions` to customize docs.rs builds. An example metadata in Cargo.toml: ```text [package] name = "test" [package.metadata.docs.rs] features = [ "feature1", "feature2" ] all-features = true no-default-features = true default-target = "x86_64-unknown-linux-gnu" rustc-args = [ "--example-rustc-arg" ] rustdoc-args = [ "--example-rustdoc-arg" ] dependencies = [ "example-system-dependency" ] ``` This patch is still work in progress and aiming to fix: #29, #48 and #50 - [ ] Save default target to database. - [ ] Install system dependencies before building a package. --- Cargo.toml | 1 + src/docbuilder/metadata.rs | 179 +++++++++++++++++++++++++++++++++++++ src/docbuilder/mod.rs | 1 + src/lib.rs | 2 + src/utils/build_doc.rs | 14 +-- 5 files changed, 192 insertions(+), 5 deletions(-) create mode 100644 src/docbuilder/metadata.rs diff --git a/Cargo.toml b/Cargo.toml index ae7aac1e7..c333901ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ libc = "0.2" hoedown = "6.0" badge = { version = "0", path = "src/web/badge" } error-chain = "0.5" +toml = "0.2" # iron dependencies iron = "0.5" diff --git a/src/docbuilder/metadata.rs b/src/docbuilder/metadata.rs new file mode 100644 index 000000000..8b5719e1a --- /dev/null +++ b/src/docbuilder/metadata.rs @@ -0,0 +1,179 @@ + +use std::path::Path; +use cargo::core::Package; +use toml; + + +/// Metadata for custom builds +/// +/// You can customize docs.rs builds by defining `[package.metadata.docs.rs]` table in your +/// crates' `Cargo.toml`. +/// +/// An example metadata: +/// +/// ```text +/// [package] +/// name = "test" +/// +/// [package.metadata.docs.rs] +/// features = [ "feature1", "feature2" ] +/// all-features = true +/// no-default-features = true +/// default-target = "x86_64-unknown-linux-gnu" +/// rustc-args = [ "--example-rustc-arg" ] +/// rustdoc-args = [ "--example-rustdoc-arg" ] +/// dependencies = [ "example-system-dependency" ] +/// ``` +/// +/// You can define one or more fields in your `Cargo.toml`. +pub struct Metadata { + /// List of features docs.rs will build. + /// + /// By default, docs.rs will only build default features. + pub features: Option>, + + /// Set `all-features` to true if you want docs.rs to build all features for your crate + pub all_features: bool, + + /// Docs.rs will always build default features. + /// + /// Set `no-default-fatures` to `false` if you want to build only certain features. + pub no_default_features: bool, + + /// Docs.rs is running on `x86_64-unknown-linux-gnu` target system and default documentation + /// is always built on this target. You can change default target by setting this. + pub default_target: Option, + + /// List of command line arguments for `rustc`. + pub rustc_args: Option>, + + /// List of command line arguments for `rustdoc`. + pub rustdoc_args: Option>, + + /// System dependencies. + /// + /// Docs.rs is running on a Debian jessie. + pub dependencies: Option>, +} + + + +impl Metadata { + pub fn from_package(pkg: &Package) -> Metadata { + Metadata::from_manifest(pkg.manifest_path()) + } + + pub fn from_manifest>(path: P) -> Metadata { + use std::fs::File; + use std::io::Read; + let mut f = match File::open(path) { + Ok(f) => f, + Err(_) => return Metadata::default(), + }; + let mut s = String::new(); + if let Err(_) = f.read_to_string(&mut s) { + return Metadata::default(); + } + Metadata::from_str(&s) + } + + + // This is similar to Default trait but it's private + fn default() -> Metadata { + Metadata { + features: None, + all_features: false, + no_default_features: false, + default_target: None, + rustc_args: None, + rustdoc_args: None, + dependencies: None, + } + } + + + fn from_str(manifest: &str) -> Metadata { + let mut metadata = Metadata::default(); + + let manifest = match toml::Parser::new(manifest).parse() { + Some(m) => m, + None => return metadata, + }; + + if let Some(table) = manifest.get("package").and_then(|p| p.as_table()) + .and_then(|p| p.get("metadata")).and_then(|p| p.as_table()) + .and_then(|p| p.get("docs")).and_then(|p| p.as_table()) + .and_then(|p| p.get("rs")).and_then(|p| p.as_table()) { + metadata.features = table.get("features").and_then(|f| f.as_slice()) + .and_then(|f| f.iter().map(|v| v.as_str().map(|v| v.to_owned())).collect()); + metadata.no_default_features = table.get("no-default-features") + .and_then(|v| v.as_bool()).unwrap_or(metadata.no_default_features); + metadata.all_features = table.get("all-features") + .and_then(|v| v.as_bool()).unwrap_or(metadata.all_features); + metadata.default_target = table.get("default-target") + .and_then(|v| v.as_str()).map(|v| v.to_owned()); + metadata.rustc_args = table.get("rustc-args").and_then(|f| f.as_slice()) + .and_then(|f| f.iter().map(|v| v.as_str().map(|v| v.to_owned())).collect()); + metadata.rustdoc_args = table.get("rustdoc-args").and_then(|f| f.as_slice()) + .and_then(|f| f.iter().map(|v| v.as_str().map(|v| v.to_owned())).collect()); + metadata.dependencies = table.get("dependencies").and_then(|f| f.as_slice()) + .and_then(|f| f.iter().map(|v| v.as_str().map(|v| v.to_owned())).collect()); + } + + metadata + } +} + + + +#[cfg(test)] +mod test { + extern crate env_logger; + use super::Metadata; + + #[test] + fn test_cratesfyi_metadata() { + let _ = env_logger::init(); + let manifest = r#" + [package] + name = "test" + + [package.metadata.docs.rs] + features = [ "feature1", "feature2" ] + all-features = true + no-default-features = true + default-target = "x86_64-unknown-linux-gnu" + rustc-args = [ "--example-rustc-arg" ] + rustdoc-args = [ "--example-rustdoc-arg" ] + dependencies = [ "example-system-dependency" ] + "#; + + let metadata = Metadata::from_str(manifest); + + assert!(metadata.features.is_some()); + assert!(metadata.all_features == true); + assert!(metadata.no_default_features == true); + assert!(metadata.default_target.is_some()); + assert!(metadata.rustc_args.is_some()); + assert!(metadata.rustdoc_args.is_some()); + + let features = metadata.features.unwrap(); + assert_eq!(features.len(), 2); + assert_eq!(features[0], "feature1".to_owned()); + assert_eq!(features[1], "feature2".to_owned()); + + assert_eq!(metadata.default_target.unwrap(), "x86_64-unknown-linux-gnu".to_owned()); + + let rustc_args = metadata.rustc_args.unwrap(); + assert_eq!(rustc_args.len(), 1); + assert_eq!(rustc_args[0], "--example-rustc-arg".to_owned()); + + let rustdoc_args = metadata.rustdoc_args.unwrap(); + assert_eq!(rustdoc_args.len(), 1); + assert_eq!(rustdoc_args[0], "--example-rustdoc-arg".to_owned()); + + let dependencies = metadata.dependencies.unwrap(); + assert_eq!(dependencies.len(), 1); + assert_eq!(dependencies[0], "example-system-dependency".to_owned()); + } +} diff --git a/src/docbuilder/mod.rs b/src/docbuilder/mod.rs index f8282a7ce..67820f143 100644 --- a/src/docbuilder/mod.rs +++ b/src/docbuilder/mod.rs @@ -1,5 +1,6 @@ pub mod options; +pub mod metadata; mod chroot_builder; mod crates; mod queue; diff --git a/src/lib.rs b/src/lib.rs index 5646d9dc3..6daf16fcb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,10 +27,12 @@ extern crate libc; extern crate badge; extern crate crates_index_diff; extern crate git2; +extern crate toml; pub use self::docbuilder::DocBuilder; pub use self::docbuilder::ChrootBuilderResult; pub use self::docbuilder::options::DocBuilderOptions; +pub use self::docbuilder::metadata::Metadata; pub use self::web::start_web_server; pub mod errors; diff --git a/src/utils/build_doc.rs b/src/utils/build_doc.rs index f39346e25..57d8675d3 100644 --- a/src/utils/build_doc.rs +++ b/src/utils/build_doc.rs @@ -12,6 +12,8 @@ use cargo::util::{CargoResult, Config, human, Filesystem}; use cargo::sources::SourceConfigMap; use cargo::ops::{self, Packages, DefaultExecutor}; +use Metadata; + /// Builds documentation of a crate and version. /// @@ -43,20 +45,22 @@ pub fn build_doc(name: &str, vers: Option<&str>, target: Option<&str>) -> CargoR let target_dir = PathBuf::from(current_dir) .join(format!("{}-{}", pkg.manifest().name(), pkg.manifest().version())); + let metadata = Metadata::from_package(&pkg); + let opts = ops::CompileOptions { config: &config, jobs: None, target: target, - features: &[], - all_features: false, - no_default_features: false, + features: &metadata.features.unwrap_or(Vec::new()), + all_features: metadata.all_features, + no_default_features: metadata.no_default_features, spec: Packages::Packages(&[]), mode: ops::CompileMode::Doc { deps: false }, release: false, message_format: ops::MessageFormat::Human, filter: ops::CompileFilter::new(true, &[], &[], &[], &[]), - target_rustc_args: None, - target_rustdoc_args: None, + target_rustc_args: metadata.rustc_args.as_ref().map(Vec::as_slice), + target_rustdoc_args: metadata.rustdoc_args.as_ref().map(Vec::as_slice), }; let ws = try!(Workspace::ephemeral(pkg, &config, Some(Filesystem::new(target_dir)))); From 8253c50133309e68a3f9d3e6b646fe37228e91af Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Sun, 5 Feb 2017 13:39:29 +0300 Subject: [PATCH 2/4] Add default target to database --- Cargo.lock | 1 + src/db/add_package.rs | 15 ++++++++++----- src/db/mod.rs | 1 + 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d8b282e21..ccbccc088 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,7 @@ dependencies = [ "staticfile 0.3.1 (git+https://github.com/onur/staticfile.git?branch=iron-0.5)", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 3a2d1df3d..3e7683070 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -1,5 +1,6 @@ use ChrootBuilderResult; +use Metadata; use utils::source_path; use regex::Regex; @@ -40,6 +41,7 @@ pub fn add_package_into_database(conn: &Connection, &TargetKind::Lib(_) => true, _ => false, }; + let metadata = Metadata::from_package(pkg); let release_id: i32 = { let rows = try!(conn.query("SELECT id FROM releases WHERE crate_id = $1 AND version = $2", @@ -53,11 +55,11 @@ pub fn add_package_into_database(conn: &Connection, homepage_url, description, description_long, readme, authors, keywords, have_examples, downloads, files, doc_targets, is_library, doc_rustc_version, - documentation_url + documentation_url, default_target ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, - $20, $21, $22, $23, $24 + $20, $21, $22, $23, $24, $25 ) RETURNING id", &[&crate_id, @@ -83,7 +85,8 @@ pub fn add_package_into_database(conn: &Connection, &doc_targets.to_json(), &is_library, &res.rustc_version, - &pkg.manifest().metadata().documentation])); + &pkg.manifest().metadata().documentation, + &metadata.default_target])); // return id rows.get(0).get(0) @@ -110,7 +113,8 @@ pub fn add_package_into_database(conn: &Connection, doc_targets = $21, is_library = $22, doc_rustc_version = $23, - documentation_url = $24 + documentation_url = $24, + default_target = $25 WHERE crate_id = $1 AND version = $2", &[&crate_id, &format!("{}", pkg.manifest().version()), @@ -135,7 +139,8 @@ pub fn add_package_into_database(conn: &Connection, &doc_targets.to_json(), &is_library, &res.rustc_version, - &pkg.manifest().metadata().documentation])); + &pkg.manifest().metadata().documentation, + &metadata.default_target])); rows.get(0).get(0) } }; diff --git a/src/db/mod.rs b/src/db/mod.rs index f188dac40..8ed55e23f 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -112,6 +112,7 @@ pub fn create_tables(conn: &Connection) -> Result<(), Error> { files JSON, doc_targets JSON DEFAULT '[]', doc_rustc_version VARCHAR(100) NOT NULL, + default_target VARCHAR(100), UNIQUE (crate_id, version) )", "CREATE TABLE authors ( From ef9e5f33307a1a3deccbe74dc7a15fda354851f6 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Thu, 16 Feb 2017 12:55:11 +0300 Subject: [PATCH 3/4] Ignore rustc_args --- src/utils/build_doc.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/utils/build_doc.rs b/src/utils/build_doc.rs index 57d8675d3..7d9b5e590 100644 --- a/src/utils/build_doc.rs +++ b/src/utils/build_doc.rs @@ -59,7 +59,10 @@ pub fn build_doc(name: &str, vers: Option<&str>, target: Option<&str>) -> CargoR release: false, message_format: ops::MessageFormat::Human, filter: ops::CompileFilter::new(true, &[], &[], &[], &[]), - target_rustc_args: metadata.rustc_args.as_ref().map(Vec::as_slice), + // FIXME: cargo is ignoring target_rustdoc_args if rustc_args set. + // I believe this is expected behavior when building docs but I am not sure. + // This needs to be investigated. + target_rustc_args: None, target_rustdoc_args: metadata.rustdoc_args.as_ref().map(Vec::as_slice), }; From 001093957bf2add76dfd78c81d6ba68dcd739ca9 Mon Sep 17 00:00:00 2001 From: Onur Aslan Date: Wed, 24 May 2017 13:45:16 +0300 Subject: [PATCH 4/4] rustc-args removed from Metadata Docs.rs is never running rustc on main crate and because of this reason, cargo is ignoring rustdoc-args when rustc-args set. Users must define rustdoc-args to manipulate rustc. --- src/docbuilder/metadata.rs | 14 +------------- src/utils/build_doc.rs | 3 --- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/src/docbuilder/metadata.rs b/src/docbuilder/metadata.rs index b1a923940..4ccabb7bb 100644 --- a/src/docbuilder/metadata.rs +++ b/src/docbuilder/metadata.rs @@ -14,13 +14,12 @@ use toml::Value; /// ```text /// [package] /// name = "test" -/// +/// /// [package.metadata.docs.rs] /// features = [ "feature1", "feature2" ] /// all-features = true /// no-default-features = true /// default-target = "x86_64-unknown-linux-gnu" -/// rustc-args = [ "--example-rustc-arg" ] /// rustdoc-args = [ "--example-rustdoc-arg" ] /// dependencies = [ "example-system-dependency" ] /// ``` @@ -44,9 +43,6 @@ pub struct Metadata { /// is always built on this target. You can change default target by setting this. pub default_target: Option, - /// List of command line arguments for `rustc`. - pub rustc_args: Option>, - /// List of command line arguments for `rustdoc`. pub rustdoc_args: Option>, @@ -85,7 +81,6 @@ impl Metadata { all_features: false, no_default_features: false, default_target: None, - rustc_args: None, rustdoc_args: None, dependencies: None, } @@ -112,8 +107,6 @@ impl Metadata { .and_then(|v| v.as_bool()).unwrap_or(metadata.all_features); metadata.default_target = table.get("default-target") .and_then(|v| v.as_str()).map(|v| v.to_owned()); - metadata.rustc_args = table.get("rustc-args").and_then(|f| f.as_array()) - .and_then(|f| f.iter().map(|v| v.as_str().map(|v| v.to_owned())).collect()); metadata.rustdoc_args = table.get("rustdoc-args").and_then(|f| f.as_array()) .and_then(|f| f.iter().map(|v| v.as_str().map(|v| v.to_owned())).collect()); metadata.dependencies = table.get("dependencies").and_then(|f| f.as_array()) @@ -154,7 +147,6 @@ mod test { assert!(metadata.all_features == true); assert!(metadata.no_default_features == true); assert!(metadata.default_target.is_some()); - assert!(metadata.rustc_args.is_some()); assert!(metadata.rustdoc_args.is_some()); let features = metadata.features.unwrap(); @@ -164,10 +156,6 @@ mod test { assert_eq!(metadata.default_target.unwrap(), "x86_64-unknown-linux-gnu".to_owned()); - let rustc_args = metadata.rustc_args.unwrap(); - assert_eq!(rustc_args.len(), 1); - assert_eq!(rustc_args[0], "--example-rustc-arg".to_owned()); - let rustdoc_args = metadata.rustdoc_args.unwrap(); assert_eq!(rustdoc_args.len(), 1); assert_eq!(rustdoc_args[0], "--example-rustdoc-arg".to_owned()); diff --git a/src/utils/build_doc.rs b/src/utils/build_doc.rs index a7eb1d034..bc9058de4 100644 --- a/src/utils/build_doc.rs +++ b/src/utils/build_doc.rs @@ -63,9 +63,6 @@ pub fn build_doc(name: &str, vers: Option<&str>, target: Option<&str>) -> CargoR &[], false, &[], false, &[], false), - // FIXME: cargo is ignoring target_rustdoc_args if rustc_args set. - // I believe this is expected behavior when building docs but I am not sure. - // This needs to be investigated. target_rustc_args: None, target_rustdoc_args: metadata.rustdoc_args.as_ref().map(Vec::as_slice), };