@@ -256,14 +236,14 @@
{{svg-jar "download"}}
- {{ format-num downloadsContext.downloads }}
+ {{format-num downloadsContext.downloads}}
Downloads all time
{{svg-jar "crate"}}
- {{ crate.versions.length }}
+ {{crate.versions.length}}
Versions published
diff --git a/ember-cli-build.js b/ember-cli-build.js
index 427a8da2746..5e987af7ed1 100644
--- a/ember-cli-build.js
+++ b/ember-cli-build.js
@@ -5,10 +5,31 @@
const EmberApp = require('ember-cli/lib/broccoli/ember-app');
module.exports = function(defaults) {
+ const highlightedLanguages = [
+ 'bash',
+ 'clike',
+ 'glsl',
+ 'go',
+ 'ini',
+ 'javascript',
+ 'json',
+ 'markup',
+ 'protobuf',
+ 'ruby',
+ 'rust',
+ 'scss',
+ 'sql',
+ 'yaml'
+ ];
+
let app = new EmberApp(defaults, {
babel6: {
plugins: ['transform-object-rest-spread'],
},
+ 'ember-prism': {
+ theme: 'twilight',
+ components: highlightedLanguages,
+ }
});
// Use `app.import` to add additional libraries to the generated
diff --git a/package-lock.json b/package-lock.json
index 88024e3203e..f6ccc2e7b01 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -6372,6 +6372,56 @@
"ember-cli-htmlbars": "2.0.2"
}
},
+ "ember-prism": {
+ "version": "0.1.0",
+ "resolved": "https://registry.npmjs.org/ember-prism/-/ember-prism-0.1.0.tgz",
+ "integrity": "sha1-JWz25nDj/9B4OXa58nSuhLU8ExQ=",
+ "dev": true,
+ "requires": {
+ "ember-cli-babel": "6.6.0",
+ "ember-cli-htmlbars": "1.3.4",
+ "ember-cli-node-assets": "0.2.2",
+ "prismjs": "1.6.0"
+ },
+ "dependencies": {
+ "ember-cli-htmlbars": {
+ "version": "1.3.4",
+ "resolved": "https://registry.npmjs.org/ember-cli-htmlbars/-/ember-cli-htmlbars-1.3.4.tgz",
+ "integrity": "sha512-5lycG6z35QHr3WZF1OkVvT+N/GGAVuemtM6m8NUgBWoeA2TqOgPFRcI0eRqoLA0HAfe0R2MReKmMI7y1LEM1+w==",
+ "dev": true,
+ "requires": {
+ "broccoli-persistent-filter": "1.4.2",
+ "ember-cli-version-checker": "1.3.1",
+ "hash-for-dep": "1.1.2",
+ "json-stable-stringify": "1.0.1",
+ "strip-bom": "2.0.0"
+ }
+ },
+ "ember-cli-node-assets": {
+ "version": "0.2.2",
+ "resolved": "https://registry.npmjs.org/ember-cli-node-assets/-/ember-cli-node-assets-0.2.2.tgz",
+ "integrity": "sha1-0tVWJufMZhn4gtf+VXUfkmYCJwg=",
+ "dev": true,
+ "requires": {
+ "broccoli-funnel": "1.2.0",
+ "broccoli-merge-trees": "1.2.4",
+ "broccoli-source": "1.1.0",
+ "debug": "2.6.8",
+ "lodash": "4.17.4",
+ "resolve": "1.3.3"
+ }
+ },
+ "ember-cli-version-checker": {
+ "version": "1.3.1",
+ "resolved": "https://registry.npmjs.org/ember-cli-version-checker/-/ember-cli-version-checker-1.3.1.tgz",
+ "integrity": "sha1-C8LRNMgwFC2mS/lieg7e0QthrnI=",
+ "dev": true,
+ "requires": {
+ "semver": "5.3.0"
+ }
+ }
+ }
+ },
"ember-qunit": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/ember-qunit/-/ember-qunit-2.1.4.tgz",
@@ -11584,6 +11634,15 @@
"integrity": "sha1-xDjKLKM+OSdnHbSracDlL5NqTw8=",
"dev": true
},
+ "prismjs": {
+ "version": "1.6.0",
+ "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.6.0.tgz",
+ "integrity": "sha1-EY2V+3pm26InLjQ7NF9SNmWds2U=",
+ "dev": true,
+ "requires": {
+ "clipboard": "1.7.1"
+ }
+ },
"private": {
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/private/-/private-0.1.7.tgz",
diff --git a/package.json b/package.json
index e164b315d03..85f2f1206a1 100644
--- a/package.json
+++ b/package.json
@@ -53,6 +53,7 @@
"ember-moment": "^7.3.1",
"ember-normalize": "^1.0.0",
"ember-page-title": "^3.2.1",
+ "ember-prism": "^0.1.0",
"ember-resolver": "^4.1.0",
"ember-router-scroll": "^0.2.0",
"ember-source": "~2.13.3",
diff --git a/src/bin/render-readmes.rs b/src/bin/render-readmes.rs
new file mode 100644
index 00000000000..00000dc3843
--- /dev/null
+++ b/src/bin/render-readmes.rs
@@ -0,0 +1,249 @@
+// Iterates over every crate versions ever uploaded and (re-)renders their
+// readme using the Markdown renderer from the cargo_registry crate.
+//
+// Warning: this can take a lot of time.
+//
+// Usage:
+// cargo run --bin render-readmes [page-size: optional = 25]
+// The page-size argument dictate how much versions should be queried and processed at once.
+
+#![deny(warnings)]
+
+#[macro_use]
+extern crate serde_derive;
+
+extern crate cargo_registry;
+extern crate curl;
+extern crate diesel;
+extern crate flate2;
+extern crate s3;
+extern crate tar;
+extern crate time;
+extern crate toml;
+extern crate url;
+
+use curl::easy::{Easy, List};
+use diesel::prelude::*;
+use flate2::read::GzDecoder;
+use std::env;
+use std::io::{Cursor, Read};
+use std::path::Path;
+use std::thread;
+use tar::Archive;
+use url::Url;
+
+use cargo_registry::{Config, Version};
+use cargo_registry::version::EncodableVersion;
+use cargo_registry::schema::*;
+use cargo_registry::render::markdown_to_html;
+
+const DEFAULT_PAGE_SIZE: i64 = 25;
+
+fn main() {
+ let config: Config = Default::default();
+ let conn = cargo_registry::db::connect_now().unwrap();
+ let versions_count = versions::table
+ .select(versions::all_columns)
+ .count()
+ .get_result::
(&conn)
+ .expect("error counting versions");
+ let page_size = match env::args().nth(1) {
+ None => DEFAULT_PAGE_SIZE,
+ Some(s) => s.parse::().unwrap_or(DEFAULT_PAGE_SIZE),
+ };
+ let pages = if versions_count % page_size == 0 {
+ versions_count / page_size
+ } else {
+ versions_count / page_size + 1
+ };
+ for current_page in 0..pages {
+ let versions: Vec = versions::table
+ .inner_join(crates::table)
+ .select((versions::all_columns, crates::name))
+ .limit(page_size)
+ .offset(current_page * page_size)
+ .load::<(Version, String)>(&conn)
+ .expect("error loading versions")
+ .into_iter()
+ .map(|(version, crate_name)| version.encodable(&crate_name))
+ .collect();
+ let mut tasks = Vec::with_capacity(page_size as usize);
+ for version in versions {
+ let config = config.clone();
+ let handle = thread::spawn(move || {
+ println!("[{}-{}] Rendering README...", version.krate, version.num);
+ let readme = get_readme(&config, &version);
+ if readme.is_none() {
+ return;
+ }
+ let readme = readme.unwrap();
+ let readme_path = format!(
+ "readmes/{}/{}-{}.html",
+ version.krate,
+ version.krate,
+ version.num
+ );
+ let readme_len = readme.len();
+ let mut body = Cursor::new(readme.into_bytes());
+ config
+ .uploader
+ .upload(
+ Easy::new(),
+ &readme_path,
+ &mut body,
+ "text/html",
+ readme_len as u64,
+ )
+ .expect(&format!(
+ "[{}-{}] Couldn't upload file to S3",
+ version.krate,
+ version.num
+ ));
+ });
+ tasks.push(handle);
+ }
+ for handle in tasks {
+ if let Err(err) = handle.join() {
+ println!("Thead panicked: {:?}", err);
+ }
+ }
+ }
+}
+
+/// Renders the readme of an uploaded crate version.
+fn get_readme(config: &Config, version: &EncodableVersion) -> Option {
+ let mut handle = Easy::new();
+ let location = match config.uploader.crate_location(&version.krate, &version.num) {
+ Some(l) => l,
+ None => return None,
+ };
+ let date = time::now().rfc822z().to_string();
+ let url = Url::parse(&location).expect(&format!(
+ "[{}-{}] Couldn't parse crate URL",
+ version.krate,
+ version.num
+ ));
+
+ let mut headers = List::new();
+ headers
+ .append(&format!("Host: {}", url.host().unwrap()))
+ .unwrap();
+ headers.append(&format!("Date: {}", date)).unwrap();
+
+ handle.url(url.as_str()).unwrap();
+ handle.get(true).unwrap();
+ handle.http_headers(headers).unwrap();
+
+ let mut response = Vec::new();
+ {
+ let mut req = handle.transfer();
+ req.write_function(|data| {
+ response.extend(data);
+ Ok(data.len())
+ }).unwrap();
+ if let Err(err) = req.perform() {
+ println!(
+ "[{}-{}] Unable to fetch crate: {}",
+ version.krate,
+ version.num,
+ err
+ );
+ return None;
+ }
+ }
+ if handle.response_code().unwrap() != 200 {
+ let response = String::from_utf8_lossy(&response);
+ println!(
+ "[{}-{}] Failed to get a 200 response: {}",
+ version.krate,
+ version.num,
+ response
+ );
+ return None;
+ }
+ let reader = Cursor::new(response);
+ let reader = GzDecoder::new(reader).expect(&format!(
+ "[{}-{}] Invalid gzip header",
+ version.krate,
+ version.num
+ ));
+ let mut archive = Archive::new(reader);
+ let mut entries = archive.entries().expect(&format!(
+ "[{}-{}] Invalid tar archive entries",
+ version.krate,
+ version.num
+ ));
+ let manifest: Manifest = {
+ let path = format!("{}-{}/Cargo.toml", version.krate, version.num);
+ let contents = find_file_by_path(&mut entries, Path::new(&path), &version);
+ toml::from_str(&contents).expect(&format!(
+ "[{}-{}] Syntax error in manifest file",
+ version.krate,
+ version.num
+ ))
+ };
+ if manifest.package.readme.is_none() {
+ return None;
+ }
+ let rendered = {
+ let path = format!(
+ "{}-{}/{}",
+ version.krate,
+ version.num,
+ manifest.package.readme.unwrap()
+ );
+ let contents = find_file_by_path(&mut entries, Path::new(&path), &version);
+ markdown_to_html(&contents).expect(&format!(
+ "[{}-{}] Couldn't render README",
+ version.krate,
+ version.num
+ ))
+ };
+ return Some(rendered);
+ #[derive(Deserialize)]
+ struct Package {
+ readme: Option,
+ }
+ #[derive(Deserialize)]
+ struct Manifest {
+ package: Package,
+ }
+}
+
+/// Search an entry by its path in a Tar archive.
+fn find_file_by_path(
+ mut entries: &mut tar::Entries,
+ path: &Path,
+ version: &EncodableVersion,
+) -> String {
+ let mut file = entries
+ .find(|entry| match *entry {
+ Err(_) => false,
+ Ok(ref file) => {
+ let filepath = match file.path() {
+ Ok(p) => p,
+ Err(_) => return false,
+ };
+ return filepath == path;
+ }
+ })
+ .expect(&format!(
+ "[{}-{}] couldn't open file: {}",
+ version.krate,
+ version.num,
+ path.display()
+ ))
+ .expect(&format!(
+ "[{}-{}] file is not present: {}",
+ version.krate,
+ version.num,
+ path.display()
+ ));
+ let mut contents = String::new();
+ file.read_to_string(&mut contents).expect(&format!(
+ "[{}-{}] Couldn't read file contents",
+ version.krate,
+ version.num
+ ));
+ contents
+}
diff --git a/src/bin/server.rs b/src/bin/server.rs
index 9245acea5ec..306d543492c 100644
--- a/src/bin/server.rs
+++ b/src/bin/server.rs
@@ -7,11 +7,10 @@ extern crate git2;
extern crate env_logger;
extern crate s3;
-use cargo_registry::{env, Env, Uploader, Replica};
+use cargo_registry::{env, Env};
use civet::Server;
use std::env;
use std::fs::{self, File};
-use std::path::PathBuf;
use std::sync::Arc;
use std::sync::mpsc::channel;
@@ -19,25 +18,25 @@ use std::sync::mpsc::channel;
fn main() {
// Initialize logging
env_logger::init().unwrap();
+ let config: cargo_registry::Config = Default::default();
// If there isn't a git checkout containing the crate index repo at the path specified
// by `GIT_REPO_CHECKOUT`, delete that directory and clone the repo specified by `GIT_REPO_URL`
// into that directory instead. Uses the credentials specified in `GIT_HTTP_USER` and
// `GIT_HTTP_PWD` via the `cargo_registry::git::credentials` function.
let url = env("GIT_REPO_URL");
- let checkout = PathBuf::from(env("GIT_REPO_CHECKOUT"));
- let repo = match git2::Repository::open(&checkout) {
+ let repo = match git2::Repository::open(&config.git_repo_checkout) {
Ok(r) => r,
Err(..) => {
- let _ = fs::remove_dir_all(&checkout);
- fs::create_dir_all(&checkout).unwrap();
+ let _ = fs::remove_dir_all(&config.git_repo_checkout);
+ fs::create_dir_all(&config.git_repo_checkout).unwrap();
let mut cb = git2::RemoteCallbacks::new();
cb.credentials(cargo_registry::git::credentials);
let mut opts = git2::FetchOptions::new();
opts.remote_callbacks(cb);
git2::build::RepoBuilder::new()
.fetch_options(opts)
- .clone(&url, &checkout)
+ .clone(&url, &config.git_repo_checkout)
.unwrap()
}
};
@@ -48,97 +47,6 @@ fn main() {
cfg.set_str("user.name", "bors").unwrap();
cfg.set_str("user.email", "bors@rust-lang.org").unwrap();
- let api_protocol = String::from("https");
-
- let mirror = if env::var("MIRROR").is_ok() {
- Replica::ReadOnlyMirror
- } else {
- Replica::Primary
- };
-
- let heroku = env::var("HEROKU").is_ok();
- let cargo_env = if heroku {
- Env::Production
- } else {
- Env::Development
- };
-
- let uploader = match (cargo_env, mirror) {
- (Env::Production, Replica::Primary) => {
- // `env` panics if these vars are not set, and in production for a primary instance,
- // that's what we want since we don't want to be able to start the server if the server
- // doesn't know where to upload crates.
- Uploader::S3 {
- bucket: s3::Bucket::new(
- env("S3_BUCKET"),
- env::var("S3_REGION").ok(),
- env("S3_ACCESS_KEY"),
- env("S3_SECRET_KEY"),
- &api_protocol,
- ),
- proxy: None,
- }
- }
- (Env::Production, Replica::ReadOnlyMirror) => {
- // Read-only mirrors don't need access key or secret key since by definition,
- // they'll only need to read from a bucket, not upload.
- //
- // Read-only mirrors might have access key or secret key, so use them if those
- // environment variables are set.
- //
- // Read-only mirrors definitely need bucket though, so that they know where
- // to serve crate files from.
- Uploader::S3 {
- bucket: s3::Bucket::new(
- env("S3_BUCKET"),
- env::var("S3_REGION").ok(),
- env::var("S3_ACCESS_KEY").unwrap_or(String::new()),
- env::var("S3_SECRET_KEY").unwrap_or(String::new()),
- &api_protocol,
- ),
- proxy: None,
- }
- }
- // In Development mode, either running as a primary instance or a read-only mirror
- _ => {
- if env::var("S3_BUCKET").is_ok() {
- // If we've set the `S3_BUCKET` variable to any value, use all of the values
- // for the related S3 environment variables and configure the app to upload to
- // and read from S3 like production does. All values except for bucket are
- // optional, like production read-only mirrors.
- println!("Using S3 uploader");
- Uploader::S3 {
- bucket: s3::Bucket::new(
- env("S3_BUCKET"),
- env::var("S3_REGION").ok(),
- env::var("S3_ACCESS_KEY").unwrap_or(String::new()),
- env::var("S3_SECRET_KEY").unwrap_or(String::new()),
- &api_protocol,
- ),
- proxy: None,
- }
- } else {
- // If we don't set the `S3_BUCKET` variable, we'll use a development-only
- // uploader that makes it possible to run and publish to a locally-running
- // crates.io instance without needing to set up an account and a bucket in S3.
- println!("Using local uploader, crate files will be in the dist directory");
- Uploader::Local
- }
- }
- };
-
- let config = cargo_registry::Config {
- uploader: uploader,
- session_key: env("SESSION_KEY"),
- git_repo_checkout: checkout,
- gh_client_id: env("GH_CLIENT_ID"),
- gh_client_secret: env("GH_CLIENT_SECRET"),
- db_url: env("DATABASE_URL"),
- env: cargo_env,
- max_upload_size: 10 * 1024 * 1024, // 10 MB default file upload size limit
- mirror: mirror,
- api_protocol: api_protocol,
- };
let app = cargo_registry::App::new(&config);
let app = cargo_registry::middleware(Arc::new(app));
@@ -147,6 +55,7 @@ fn main() {
let categories_toml = include_str!("../categories.toml");
cargo_registry::categories::sync(&categories_toml).unwrap();
+ let heroku = env::var("HEROKU").is_ok();
let port = if heroku {
8888
} else {
@@ -155,7 +64,11 @@ fn main() {
.and_then(|s| s.parse().ok())
.unwrap_or(8888)
};
- let threads = if cargo_env == Env::Development { 1 } else { 50 };
+ let threads = if config.env == Env::Development {
+ 1
+ } else {
+ 50
+ };
let mut cfg = civet::Config::new();
cfg.port(port).threads(threads).keep_alive(true);
let _a = Server::start(cfg, app);
diff --git a/src/config.rs b/src/config.rs
index 30890add680..40f91f5611b 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -1,5 +1,9 @@
+use s3;
+
+use std::env;
use std::path::PathBuf;
-use {Uploader, Replica};
+
+use {Env, env, Uploader, Replica};
#[derive(Clone, Debug)]
pub struct Config {
@@ -14,3 +18,115 @@ pub struct Config {
pub mirror: Replica,
pub api_protocol: String,
}
+
+impl Default for Config {
+ /// Returns a default value for the application's config
+ ///
+ /// Sets the following default values:
+ /// - `Config::max_upload_size`: 10MiB
+ /// - `Config::api_protocol`: `https`
+ ///
+ /// Pulls values from the following environment variables:
+ /// - `GIT_REPO_CHECKOUT`: The directory where the registry index was cloned.
+ /// - `MIRROR`: Is this instance of cargo_registry a mirror of crates.io.
+ /// - `HEROKU`: Is this instance of cargo_registry currently running on Heroku.
+ /// - `S3_BUCKET`: The S3 bucket used to store crate files. If not present during development,
+ /// cargo_registry will fall back to a local uploader.
+ /// - `S3_REGION`: The region in which the bucket was created. Optional if US standard.
+ /// - `S3_ACCESS_KEY`: The access key to interact with S3. Optional if running a mirror.
+ /// - `S3_SECRET_KEY`: The secret key to interact with S3. Optional if running a mirror.
+ /// - `SESSION_KEY`: The key used to sign and encrypt session cookies.
+ /// - `GH_CLIENT_ID`: The client ID of the associated GitHub application.
+ /// - `GH_CLIENT_SECRET`: The client secret of the associated GitHub application.
+ /// - `DATABASE_URL`: The URL of the postgres database to use.
+ fn default() -> Config {
+ let checkout = PathBuf::from(env("GIT_REPO_CHECKOUT"));
+ let api_protocol = String::from("https");
+ let mirror = if env::var("MIRROR").is_ok() {
+ Replica::ReadOnlyMirror
+ } else {
+ Replica::Primary
+ };
+ let heroku = env::var("HEROKU").is_ok();
+ let cargo_env = if heroku {
+ Env::Production
+ } else {
+ Env::Development
+ };
+ let uploader = match (cargo_env, mirror) {
+ (Env::Production, Replica::Primary) => {
+ // `env` panics if these vars are not set, and in production for a primary instance,
+ // that's what we want since we don't want to be able to start the server if the
+ // server doesn't know where to upload crates.
+ Uploader::S3 {
+ bucket: s3::Bucket::new(
+ env("S3_BUCKET"),
+ env::var("S3_REGION").ok(),
+ env("S3_ACCESS_KEY"),
+ env("S3_SECRET_KEY"),
+ &api_protocol,
+ ),
+ proxy: None,
+ }
+ }
+ (Env::Production, Replica::ReadOnlyMirror) => {
+ // Read-only mirrors don't need access key or secret key since by definition,
+ // they'll only need to read from a bucket, not upload.
+ //
+ // Read-only mirrors might have access key or secret key, so use them if those
+ // environment variables are set.
+ //
+ // Read-only mirrors definitely need bucket though, so that they know where
+ // to serve crate files from.
+ Uploader::S3 {
+ bucket: s3::Bucket::new(
+ env("S3_BUCKET"),
+ env::var("S3_REGION").ok(),
+ env::var("S3_ACCESS_KEY").unwrap_or_default(),
+ env::var("S3_SECRET_KEY").unwrap_or_default(),
+ &api_protocol,
+ ),
+ proxy: None,
+ }
+ }
+ // In Development mode, either running as a primary instance or a read-only mirror
+ _ => {
+ if env::var("S3_BUCKET").is_ok() {
+ // If we've set the `S3_BUCKET` variable to any value, use all of the values
+ // for the related S3 environment variables and configure the app to upload to
+ // and read from S3 like production does. All values except for bucket are
+ // optional, like production read-only mirrors.
+ println!("Using S3 uploader");
+ Uploader::S3 {
+ bucket: s3::Bucket::new(
+ env("S3_BUCKET"),
+ env::var("S3_REGION").ok(),
+ env::var("S3_ACCESS_KEY").unwrap_or_default(),
+ env::var("S3_SECRET_KEY").unwrap_or_default(),
+ &api_protocol,
+ ),
+ proxy: None,
+ }
+ } else {
+ // If we don't set the `S3_BUCKET` variable, we'll use a development-only
+ // uploader that makes it possible to run and publish to a locally-running
+ // crates.io instance without needing to set up an account and a bucket in S3.
+ println!("Using local uploader, crate files will be in the dist directory");
+ Uploader::Local
+ }
+ }
+ };
+ Config {
+ uploader: uploader,
+ session_key: env("SESSION_KEY"),
+ git_repo_checkout: checkout,
+ gh_client_id: env("GH_CLIENT_ID"),
+ gh_client_secret: env("GH_CLIENT_SECRET"),
+ db_url: env("DATABASE_URL"),
+ env: cargo_env,
+ max_upload_size: 10 * 1024 * 1024, // 10 MB default file upload size limit
+ mirror: mirror,
+ api_protocol: api_protocol,
+ }
+ }
+}
diff --git a/src/http.rs b/src/http.rs
index 552bca78913..13a7cc234c0 100644
--- a/src/http.rs
+++ b/src/http.rs
@@ -1,14 +1,21 @@
use conduit::{Request, Response};
use conduit_middleware::Middleware;
+
use curl;
use curl::easy::{Easy, List};
+
use oauth2::*;
-use app::App;
-use util::{CargoResult, internal, ChainError, human};
+
use serde_json;
use serde::Deserialize;
+
use std::str;
use std::error::Error;
+use std::collections::HashMap;
+
+use app::App;
+use util::{CargoResult, internal, ChainError, human};
+use Uploader;
/// Does all the nonsense for sending a GET to Github. Doesn't handle parsing
/// because custom error-code handling may be desirable. Use
@@ -91,8 +98,55 @@ pub fn token(token: String) -> Token {
}
}
-#[derive(Clone, Copy, Debug)]
-pub struct SecurityHeadersMiddleware;
+#[derive(Clone, Debug)]
+pub struct SecurityHeadersMiddleware {
+ headers: HashMap>,
+}
+
+impl SecurityHeadersMiddleware {
+ pub fn new(uploader: &Uploader) -> Self {
+ let mut headers = HashMap::new();
+
+ headers.insert("X-Content-Type-Options".into(), vec!["nosniff".into()]);
+
+ headers.insert("X-Frame-Options".into(), vec!["SAMEORIGIN".into()]);
+
+ headers.insert("X-XSS-Protection".into(), vec!["1; mode=block".into()]);
+
+ let s3_host = match *uploader {
+ Uploader::S3 { ref bucket, .. } => bucket.host(),
+ _ => {
+ unreachable!(
+ "This middleware should only be used in the production environment, \
+ which should also require an S3 uploader, QED"
+ )
+ }
+ };
+
+ // It would be better if we didn't have to have 'unsafe-eval' in the `script-src`
+ // policy, but google charts (used for the download graph on crate pages) uses `eval`
+ // to load scripts. Remove 'unsafe-eval' if google fixes the issue:
+ // https://github.com/google/google-visualization-issues/issues/1356
+ // or if we switch to a different graph generation library.
+ headers.insert(
+ "Content-Security-Policy".into(),
+ vec![
+ format!(
+ "default-src 'self'; \
+ connect-src 'self' https://docs.rs https://{}; \
+ script-src 'self' 'unsafe-eval' \
+ https://www.google-analytics.com https://www.google.com; \
+ style-src 'self' https://www.google.com https://ajax.googleapis.com; \
+ img-src *; \
+ object-src 'none'",
+ s3_host
+ ),
+ ],
+ );
+
+ SecurityHeadersMiddleware { headers }
+ }
+}
impl Middleware for SecurityHeadersMiddleware {
fn after(
@@ -101,36 +155,7 @@ impl Middleware for SecurityHeadersMiddleware {
mut res: Result>,
) -> Result> {
if let Ok(ref mut response) = res {
- // It would be better if we didn't have to have 'unsafe-eval' in the `script-src`
- // policy, but google charts (used for the download graph on crate pages) uses `eval`
- // to load scripts. Remove 'unsafe-eval' if google fixes the issue:
- // https://github.com/google/google-visualization-issues/issues/1356
- // or if we switch to a different graph generation library.
- response.headers.insert(
- "Content-Security-Policy".into(),
- vec![
- "default-src 'self'; \
- connect-src 'self' https://docs.rs; \
- script-src 'self' 'unsafe-eval' \
- https://www.google-analytics.com https://www.google.com; \
- style-src 'self' https://www.google.com https://ajax.googleapis.com; \
- img-src *; \
- object-src 'none'"
- .into(),
- ],
- );
- response.headers.insert(
- "X-Content-Type-Options".into(),
- vec!["nosniff".into()],
- );
- response.headers.insert(
- "X-Frame-Options".into(),
- vec!["SAMEORIGIN".into()],
- );
- response.headers.insert(
- "X-XSS-Protection".into(),
- vec!["1; mode=block".into()],
- );
+ response.headers.extend(self.headers.clone());
}
res
}
diff --git a/src/krate.rs b/src/krate.rs
index 7859b710e97..2da5fe23438 100644
--- a/src/krate.rs
+++ b/src/krate.rs
@@ -29,6 +29,7 @@ use git;
use keyword::{EncodableKeyword, CrateKeyword};
use owner::{EncodableOwner, Owner, Rights, OwnerKind, Team, rights, CrateOwner};
use pagination::Paginate;
+use render;
use schema::*;
use upload;
use user::RequestUser;
@@ -951,10 +952,22 @@ pub fn new(req: &mut Request) -> CargoResult {
let ignored_invalid_badges = Badge::update_crate(&conn, &krate, new_crate.badges.as_ref())?;
let max_version = krate.max_version(&conn)?;
+ // Render the README for this crate
+ let readme = match new_crate.readme.as_ref() {
+ Some(readme) => Some(render::markdown_to_html(&**readme)?),
+ None => None,
+ };
+
// Upload the crate, return way to delete the crate from the server
// If the git commands fail below, we shouldn't keep the crate on the
// server.
- let (cksum, mut bomb) = app.config.uploader.upload(req, &krate, max, vers)?;
+ let (cksum, mut crate_bomb, mut readme_bomb) = app.config.uploader.upload_crate(
+ req,
+ &krate,
+ readme,
+ max,
+ vers,
+ )?;
// Register this crate in our local git repo.
let git_crate = git::Crate {
@@ -973,7 +986,8 @@ pub fn new(req: &mut Request) -> CargoResult {
})?;
// Now that we've come this far, we're committed!
- bomb.path = None;
+ crate_bomb.path = None;
+ readme_bomb.path = None;
#[derive(Serialize)]
struct Warnings<'a> {
@@ -1074,6 +1088,28 @@ pub fn download(req: &mut Request) -> CargoResult {
}
}
+/// Handles the `GET /crates/:crate_id/:version/readme` route.
+pub fn readme(req: &mut Request) -> CargoResult {
+ let crate_name = &req.params()["crate_id"];
+ let version = &req.params()["version"];
+
+ let redirect_url = req.app()
+ .config
+ .uploader
+ .readme_location(crate_name, version)
+ .ok_or_else(|| human("crate readme not found"))?;
+
+ if req.wants_json() {
+ #[derive(Serialize)]
+ struct R {
+ url: String,
+ }
+ Ok(req.json(&R { url: redirect_url }))
+ } else {
+ Ok(req.redirect(redirect_url))
+ }
+}
+
fn increment_download_counts(req: &Request, crate_name: &str, version: &str) -> CargoResult<()> {
use self::versions::dsl::*;
diff --git a/src/lib.rs b/src/lib.rs
index 0f7f8bf8dac..f13cc6ae250 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -19,6 +19,7 @@ extern crate log;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;
+extern crate ammonia;
extern crate chrono;
extern crate curl;
extern crate diesel_full_text_search;
@@ -29,6 +30,7 @@ extern crate hex;
extern crate license_exprs;
extern crate oauth2;
extern crate openssl;
+extern crate pulldown_cmark;
extern crate r2d2;
extern crate r2d2_diesel;
extern crate rand;
@@ -84,6 +86,7 @@ pub mod http;
pub mod keyword;
pub mod krate;
pub mod owner;
+pub mod render;
pub mod schema;
pub mod token;
pub mod upload;
@@ -135,6 +138,7 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder {
api_router.put("/crates/new", C(krate::new));
api_router.get("/crates/:crate_id/:version", C(version::show));
api_router.get("/crates/:crate_id/:version/download", C(krate::download));
+ api_router.get("/crates/:crate_id/:version/readme", C(krate::readme));
api_router.get(
"/crates/:crate_id/:version/dependencies",
C(version::dependencies),
@@ -224,7 +228,7 @@ pub fn middleware(app: Arc) -> MiddlewareBuilder {
env == Env::Production,
));
if env == Env::Production {
- m.add(http::SecurityHeadersMiddleware);
+ m.add(http::SecurityHeadersMiddleware::new(&app.config.uploader));
}
m.add(app::AppMiddleware::new(app));
diff --git a/src/render.rs b/src/render.rs
new file mode 100644
index 00000000000..96659be002e
--- /dev/null
+++ b/src/render.rs
@@ -0,0 +1,170 @@
+use ammonia::Ammonia;
+use pulldown_cmark::Parser;
+use pulldown_cmark::html;
+
+use util::CargoResult;
+
+/// Context for markdown to HTML rendering.
+#[allow(missing_debug_implementations)]
+pub struct MarkdownRenderer<'a> {
+ html_sanitizer: Ammonia<'a>,
+}
+
+impl<'a> MarkdownRenderer<'a> {
+ /// Creates a new renderer instance.
+ pub fn new() -> MarkdownRenderer<'a> {
+ let tags = [
+ "a",
+ "b",
+ "blockquote",
+ "br",
+ "code",
+ "dd",
+ "del",
+ "dl",
+ "dt",
+ "em",
+ "i",
+ "h1",
+ "h2",
+ "h3",
+ "hr",
+ "img",
+ "kbd",
+ "li",
+ "ol",
+ "p",
+ "pre",
+ "s",
+ "strike",
+ "strong",
+ "sub",
+ "sup",
+ "table",
+ "tbody",
+ "td",
+ "th",
+ "thead",
+ "tr",
+ "ul",
+ "hr",
+ "span",
+ ].iter()
+ .cloned()
+ .collect();
+ let tag_attributes = [
+ ("a", ["href", "target"].iter().cloned().collect()),
+ (
+ "img",
+ ["width", "height", "src", "alt", "align", "width"]
+ .iter()
+ .cloned()
+ .collect(),
+ ),
+ ("code", ["class"].iter().cloned().collect()),
+ ("span", ["style"].iter().cloned().collect()),
+ ].iter()
+ .cloned()
+ .collect();
+ let html_sanitizer = Ammonia {
+ keep_cleaned_elements: true,
+ tags: tags,
+ tag_attributes: tag_attributes,
+ ..Ammonia::default()
+ };
+ MarkdownRenderer { html_sanitizer: html_sanitizer }
+ }
+
+ /// Renders the given markdown to HTML using the current settings.
+ pub fn to_html(&self, text: &str) -> CargoResult {
+ let mut rendered = String::with_capacity(text.len() * 3 / 2);
+ let parser = Parser::new(text);
+ html::push_html(&mut rendered, parser);
+ Ok(self.html_sanitizer.clean(&rendered))
+ }
+}
+
+impl<'a> Default for MarkdownRenderer<'a> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+/// Renders a markdown text to sanitized HTML.
+///
+/// The returned text should not contain any harmful HTML tag or attribute (such as iframe,
+/// onclick, onmouseover, etc.).
+///
+/// # Examples
+///
+/// ```
+/// use render::markdown_to_html;
+///
+/// let text = "[Rust](https://rust-lang.org/) is an awesome *systems programming* language!";
+/// let rendered = markdown_to_html(text)?;
+/// ```
+pub fn markdown_to_html(text: &str) -> CargoResult {
+ let renderer = MarkdownRenderer::new();
+ renderer.to_html(text)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn empty_text() {
+ let text = "";
+ let result = markdown_to_html(text);
+ assert_eq!(result.is_ok(), true);
+ let rendered = result.unwrap();
+ assert_eq!(rendered, "");
+ }
+
+ #[test]
+ fn text_with_script_tag() {
+ let text = "foo_readme\n\n";
+ let result = markdown_to_html(text);
+ assert_eq!(result.is_ok(), true);
+ let rendered = result.unwrap();
+ assert_eq!(rendered.contains("foo_readme"), true);
+ assert_eq!(rendered.contains("script"), false);
+ assert_eq!(rendered.contains("alert('Hello World')"), true);
+ }
+
+ #[test]
+ fn text_with_iframe_tag() {
+ let text = "foo_readme\n\n";
+ let result = markdown_to_html(text);
+ assert_eq!(result.is_ok(), true);
+ let rendered = result.unwrap();
+ assert_eq!(rendered.contains("foo_readme"), true);
+ assert_eq!(rendered.contains("iframe"), false);
+ assert_eq!(rendered.contains("alert('Hello World')"), true);
+ }
+
+ #[test]
+ fn text_with_unknwon_tag() {
+ let text = "foo_readme\n\nalert('Hello World')";
+ let result = markdown_to_html(text);
+ assert_eq!(result.is_ok(), true);
+ let rendered = result.unwrap();
+ assert_eq!(rendered.contains("foo_readme"), true);
+ assert_eq!(rendered.contains("unknown"), false);
+ assert_eq!(rendered.contains("alert('Hello World')"), true);
+ }
+
+ #[test]
+ fn text_with_inline_javascript() {
+ let text = r#"foo_readme\n\nCrate page"#;
+ let result = markdown_to_html(text);
+ assert_eq!(result.is_ok(), true);
+ let rendered = result.unwrap();
+ assert_eq!(rendered.contains("foo_readme"), true);
+ assert_eq!(rendered.contains(" Option {
match *self {
Uploader::S3 { ref bucket, .. } => {
@@ -54,35 +60,60 @@ impl Uploader {
}
}
+ /// Returns the URL of an uploaded crate's version readme.
+ ///
+ /// The function doesn't check for the existence of the file.
+ /// It returns `None` if the current `Uploader` is `NoOp`.
+ pub fn readme_location(&self, crate_name: &str, version: &str) -> Option {
+ match *self {
+ Uploader::S3 { ref bucket, .. } => {
+ Some(format!(
+ "https://{}/{}",
+ bucket.host(),
+ Uploader::readme_path(crate_name, version)
+ ))
+ }
+ Uploader::Local => {
+ Some(format!(
+ "/local_uploads/{}",
+ Uploader::readme_path(crate_name, version)
+ ))
+ }
+ Uploader::NoOp => None,
+ }
+ }
+
+ /// Returns the interna path of an uploaded crate's version archive.
fn crate_path(name: &str, version: &str) -> String {
// No slash in front so we can use join
format!("crates/{}/{}-{}.crate", name, name, version)
}
+ /// Returns the interna path of an uploaded crate's version readme.
+ fn readme_path(name: &str, version: &str) -> String {
+ format!("readmes/{}/{}-{}.html", name, name, version)
+ }
+
+ /// Uploads a file using the configured uploader (either `S3`, `Local` or `NoOp`).
+ ///
+ /// It returns a a tuple containing the path of the uploaded file
+ /// and its checksum.
pub fn upload(
&self,
- req: &mut Request,
- krate: &Crate,
- max: u64,
- vers: &semver::Version,
- ) -> CargoResult<(Vec, Bomb)> {
+ mut handle: Easy,
+ path: &str,
+ body: &mut io::Read,
+ content_type: &str,
+ content_length: u64,
+ ) -> CargoResult<(Option, Vec)> {
match *self {
Uploader::S3 { ref bucket, .. } => {
- let mut handle = req.app().handle();
- let path = format!("/{}", Uploader::crate_path(&krate.name, &vers.to_string()));
let (response, cksum) = {
- let length = read_le_u32(req.body())?;
- let body = LimitErrorReader::new(req.body(), max);
let mut body = HashingReader::new(body);
let mut response = Vec::new();
{
- let mut s3req = bucket.put(
- &mut handle,
- &path,
- &mut body,
- "application/x-tar",
- length as u64,
- );
+ let mut s3req =
+ bucket.put(&mut handle, path, &mut body, content_type, content_length);
s3req
.write_function(|data| {
response.extend(data);
@@ -102,57 +133,79 @@ impl Uploader {
response
)));
}
-
- Ok((
- cksum,
- Bomb {
- app: req.app().clone(),
- path: Some(path),
- },
- ))
+ Ok((Some(String::from(path)), cksum))
}
Uploader::Local => {
- let path = Uploader::crate_path(&krate.name, &vers.to_string());
- let crate_filename = env::current_dir()
+ let filename = env::current_dir()
.unwrap()
.join("dist")
.join("local_uploads")
.join(path);
-
- let crate_dir = crate_filename.parent().unwrap();
- fs::create_dir_all(crate_dir)?;
-
- let mut crate_file = File::create(&crate_filename)?;
-
- let cksum = {
- read_le_u32(req.body())?;
- let body = LimitErrorReader::new(req.body(), max);
- let mut body = HashingReader::new(body);
-
- io::copy(&mut body, &mut crate_file)?;
- body.finalize()
- };
-
- Ok((
- cksum,
- Bomb {
- app: req.app().clone(),
- path: crate_filename.to_str().map(String::from),
- },
- ))
- }
- Uploader::NoOp => {
- Ok((
- vec![],
- Bomb {
- app: req.app().clone(),
- path: None,
- },
- ))
+ let dir = filename.parent().unwrap();
+ fs::create_dir_all(dir)?;
+ let mut file = File::create(&filename)?;
+ let mut body = HashingReader::new(body);
+ io::copy(&mut body, &mut file)?;
+ Ok((filename.to_str().map(String::from), body.finalize()))
}
+ Uploader::NoOp => Ok((None, vec![])),
}
}
+ /// Uploads a crate and its readme. Returns the checksum of the uploaded crate
+ /// file, and bombs for the uploaded crate and the uploaded readme.
+ pub fn upload_crate(
+ &self,
+ req: &mut Request,
+ krate: &Crate,
+ readme: Option,
+ max: u64,
+ vers: &semver::Version,
+ ) -> CargoResult<(Vec, Bomb, Bomb)> {
+ let app = req.app().clone();
+ let (crate_path, checksum) = {
+ let path = Uploader::crate_path(&krate.name, &vers.to_string());
+ let length = read_le_u32(req.body())?;
+ let mut body = LimitErrorReader::new(req.body(), max);
+ self.upload(
+ app.handle(),
+ &path,
+ &mut body,
+ "application/x-tar",
+ length as u64,
+ )?
+ };
+ // We create the bomb for the crate file before uploading the readme so that if the
+ // readme upload fails, the uploaded crate file is automatically deleted.
+ let crate_bomb = Bomb {
+ app: app.clone(),
+ path: crate_path,
+ };
+ let (readme_path, _) = if let Some(rendered) = readme {
+ let path = Uploader::readme_path(&krate.name, &vers.to_string());
+ let length = rendered.len();
+ let mut body = io::Cursor::new(rendered.into_bytes());
+ self.upload(
+ app.handle(),
+ &path,
+ &mut body,
+ "text/html",
+ length as u64,
+ )?
+ } else {
+ (None, vec![])
+ };
+ Ok((
+ checksum,
+ crate_bomb,
+ Bomb {
+ app: app.clone(),
+ path: readme_path,
+ },
+ ))
+ }
+
+ /// Deletes an uploaded file.
fn delete(&self, app: Arc, path: &str) -> CargoResult<()> {
match *self {
Uploader::S3 { ref bucket, .. } => {
diff --git a/src/version.rs b/src/version.rs
index e45f1209935..1461d2df6d4 100644
--- a/src/version.rs
+++ b/src/version.rs
@@ -53,6 +53,7 @@ pub struct EncodableVersion {
pub krate: String,
pub num: String,
pub dl_path: String,
+ pub readme_path: String,
pub updated_at: String,
pub created_at: String,
pub downloads: i32,
@@ -85,6 +86,7 @@ impl Version {
let num = num.to_string();
EncodableVersion {
dl_path: format!("/api/v1/crates/{}/{}/download", crate_name, num),
+ readme_path: format!("/api/v1/crates/{}/{}/readme", crate_name, num),
num: num.clone(),
id: id,
krate: crate_name.to_string(),