rustdoc: Migrate from sundown to hoedown

This primary fix brought on by this upgrade is the proper matching of the ```
and ~~~ doc blocks. This also moves hoedown to a git submodule rather than a
bundled repository.

Additionally, hoedown is stricter about code blocks, so this ended up fixing a
lot of invalid code blocks (ending with " ```" instead of "```", or ending with
"~~~~" instead of "~~~").

Closes #12776

Author: alexcrichton

.gitmodules

@@ -12,3 +12,6 @@
 [submodule "src/compiler-rt"]
 	path = src/compiler-rt
 	url = https://github.com/rust-lang/compiler-rt.git
+[submodule "src/rt/hoedown"]
+	path = src/rt/hoedown
+	url = https://github.com/rust-lang/hoedown.git

mk/crates.mk

@@ -38,7 +38,7 @@
 #   DEPS_<crate>
 #	These lists are the dependencies of the <crate> that is to be built.
 #	Rust dependencies are listed bare (i.e. std, green) and native
-#	dependencies have a "native:" prefix (i.e. native:sundown). All deps
+#	dependencies have a "native:" prefix (i.e. native:hoedown). All deps
 #	will be built before the crate itself is built.
 #
 #   TOOL_DEPS_<tool>/TOOL_SOURCE_<tool>
@@ -63,7 +63,7 @@ DEPS_native := std
 DEPS_syntax := std term serialize collections log
 DEPS_rustc := syntax native:rustllvm flate arena serialize sync getopts \
               collections time log
-DEPS_rustdoc := rustc native:sundown serialize sync getopts collections \
+DEPS_rustdoc := rustc native:hoedown serialize sync getopts collections \
                 test time
 DEPS_flate := std native:miniz
 DEPS_arena := std collections

mk/dist.mk

@@ -35,7 +35,8 @@ LICENSE.txt: $(S)COPYRIGHT $(S)LICENSE-APACHE $(S)LICENSE-MIT
 
 PKG_TAR = dist/$(PKG_NAME).tar.gz
 
-PKG_GITMODULES := $(S)src/libuv $(S)src/llvm $(S)src/gyp $(S)src/compiler-rt
+PKG_GITMODULES := $(S)src/libuv $(S)src/llvm $(S)src/gyp $(S)src/compiler-rt \
+		  $(S)src/rt/hoedown
 PKG_FILES := \
     $(S)COPYRIGHT                              \
     $(S)LICENSE-APACHE                         \

mk/rt.mk

@@ -35,19 +35,20 @@
 # that's per-target so you're allowed to conditionally add files based on the
 # target.
 ################################################################################
-NATIVE_LIBS := rustrt sundown uv_support morestack miniz context_switch
+NATIVE_LIBS := rustrt hoedown uv_support morestack miniz context_switch
 
 # $(1) is the target triple
 define NATIVE_LIBRARIES
 
-NATIVE_DEPS_sundown_$(1) := sundown/src/autolink.c \
-			sundown/src/buffer.c \
-			sundown/src/stack.c \
-			sundown/src/markdown.c \
-			sundown/html/houdini_href_e.c \
-			sundown/html/houdini_html_e.c \
-			sundown/html/html_smartypants.c \
-			sundown/html/html.c
+NATIVE_DEPS_hoedown_$(1) := hoedown/src/autolink.c \
+			hoedown/src/buffer.c \
+			hoedown/src/document.c \
+			hoedown/src/escape.c \
+			hoedown/src/html.c \
+			hoedown/src/html_blocks.c \
+			hoedown/src/html_smartypants.c \
+			hoedown/src/stack.c \
+			hoedown/src/version.c
 NATIVE_DEPS_uv_support_$(1) := rust_uv.c
 NATIVE_DEPS_miniz_$(1) = miniz.c
 NATIVE_DEPS_rustrt_$(1) := rust_builtin.c \
@@ -79,7 +80,7 @@ $$(RT_OUTPUT_DIR_$(1))/%.o: $(S)src/rt/%.c $$(MKFILE_DEPS)
 	@mkdir -p $$(@D)
 	@$$(call E, compile: $$@)
 	$$(Q)$$(call CFG_COMPILE_C_$(1), $$@, \
-		-I $$(S)src/rt/sundown/src -I $$(S)src/rt/sundown/html \
+		-I $$(S)src/rt/hoedown/src \
 		-I $$(S)src/libuv/include -I $$(S)src/rt \
                  $$(RUNTIME_CFLAGS_$(1))) $$<
 

src/doc/complement-cheatsheet.md

@@ -62,7 +62,7 @@ use std::str;
 
 let x: Option<~str> = str::from_utf8_owned(~[104u8,105u8]);
 let y: ~str = x.unwrap();
-~~~~
+~~~
 
 To return a [`MaybeOwned`](http://static.rust-lang.org/doc/master/std/str/enum.MaybeOwned.html) use the str helper function [`from_utf8_lossy`](http://static.rust-lang.org/doc/master/std/str/fn.from_utf8_owned.html).  This function also replaces non-valid utf-8 sequences with U+FFFD replacement character.
 
@@ -71,7 +71,7 @@ use std::str;
 
 let x = bytes!(72u8,"ello ",0xF0,0x90,0x80,"World!");
 let y = str::from_utf8_lossy(x);
-~~~~
+~~~
 
 # File operations
 

src/doc/complement-lang-faq.md

@@ -135,7 +135,7 @@ For simplicity, we do not plan to do so. Implementing automatic semicolon insert
 
 **Short answer** set the RUST_LOG environment variable to the name of your source file, sans extension.
 
-``` {.sh .notrust}
+```notrust,sh
 rustc hello.rs
 export RUST_LOG=hello
 ./hello

src/doc/guide-container.md

@@ -81,7 +81,7 @@ impl Iterator<int> for ZeroStream {
         Some(0)
     }
 }
-~~~~
+~~~
 
 Reaching the end of the iterator is signalled by returning `None` instead of
 `Some(item)`:

src/doc/guide-tasks.md

@@ -496,7 +496,7 @@ fn stringifier(channel: &sync::DuplexStream<~str, uint>) {
     }
 }
 # }
-~~~~
+~~~
 
 The implementation of `DuplexStream` supports both sending and
 receiving. The `stringifier` function takes a `DuplexStream` that can
@@ -538,7 +538,7 @@ assert!(from_child.recv() == "23".to_owned());
 assert!(from_child.recv() == "0".to_owned());
 
 # }
-~~~~
+~~~
 
 The parent task first calls `DuplexStream` to create a pair of bidirectional
 endpoints. It then uses `task::spawn` to create the child task, which captures

src/doc/rust.md

@@ -4085,7 +4085,7 @@ fn main() {
 
 These four log levels correspond to levels 1-4, as controlled by `RUST_LOG`:
 
-``` {.bash .notrust}
+```notrust,bash
 $ RUST_LOG=rust=3 ./rust
 This is an error log
 This is a warn log

src/doc/rustdoc.md

@@ -107,68 +107,55 @@ code blocks as testable-by-default. In order to not run a test over a block of
 code, the `ignore` string can be added to the three-backtick form of markdown
 code block.
 
-    /**
-    # nested code fences confuse sundown => indentation + comment to
-    #  avoid failing tests
-    ```
-    // This is a testable code block
-    ```
+~~~notrust
+```
+// This is a testable code block
+```
 
-    ```ignore
-    // This is not a testable code block
-    ```
+```ignore
+// This is not a testable code block
+```
 
-        // This is a testable code block (4-space indent)
-    */
-    # fn foo() {}
+    // This is a testable code block (4-space indent)
+~~~
 
 You can specify that the test's execution should fail with the `should_fail`
 directive.
 
-    /**
-    # nested code fences confuse sundown => indentation + comment to
-    #  avoid failing tests
-    ```should_fail
-    // This code block is expected to generate a failure when run
-    ```
-    */
-    # fn foo() {}
+~~~notrust
+```should_fail
+// This code block is expected to generate a failure when run
+```
+~~~
 
 You can specify that the code block should be compiled but not run with the
 `no_run` directive.
 
-    /**
-    # nested code fences confuse sundown => indentation + comment to
-    #  avoid failing tests
-    ```no_run
-    // This code will be compiled but not executed
-    ```
-    */
-    # fn foo() {}
+~~~notrust
+```no_run
+// This code will be compiled but not executed
+```
+~~~
 
 Rustdoc also supplies some extra sugar for helping with some tedious
 documentation examples. If a line is prefixed with `# `, then the line
 will not show up in the HTML documentation, but it will be used when
 testing the code block (NB. the space after the `#` is required, so
 that one can still write things like `#[deriving(Eq)]`).
 
-    /**
-    # nested code fences confuse sundown => indentation + comment to
-    #  avoid failing tests
-    ```rust
-    # /!\ The three following lines are comments, which are usually stripped off by
-    # the doc-generating tool.  In order to display them anyway in this particular
-    # case, the character following the leading '#' is not a usual space like in
-    # these first five lines but a non breakable one.
-    #
-    # // showing 'fib' in this documentation would just be tedious and detracts from
-    # // what's actually being documented.
-    # fn fib(n: int) { n + 2 }
-
-    spawn(proc() { fib(200); })
-    ```
-    */
-    # fn foo() {}
+~~~notrust
+```
+# /!\ The three following lines are comments, which are usually stripped off by
+# the doc-generating tool.  In order to display them anyway in this particular
+# case, the character following the leading '#' is not a usual space like in
+# these first five lines but a non breakable one.
+# // showing 'fib' in this documentation would just be tedious and detracts from
+# // what's actually being documented.
+# fn fib(n: int) { n + 2 }
+
+spawn(proc() { fib(200); })
+```
+~~~
 
 The documentation online would look like `spawn(proc() { fib(200); })`, but when
 testing this code, the `fib` function will be included (so it can compile).
@@ -182,10 +169,10 @@ rustc's `--test` flag. Extra arguments can be passed to rustdoc's test harness
 with the `--test-args` flag.
 
 ~~~ {.notrust}
-$ # Only run tests containing 'foo' in their name
+# Only run tests containing 'foo' in their name
 $ rustdoc --test lib.rs --test-args 'foo'
 
-$ # See what's possible when running tests
+# See what's possible when running tests
 $ rustdoc --test lib.rs --test-args '--help'
 ~~~
 

src/doc/tutorial.md

@@ -1408,7 +1408,7 @@ struct Point {
     x: f64,
     y: f64
 }
-~~~~
+~~~
 
 We can use this simple definition to allocate points in many different
 ways. For example, in this code, each of these local variables

src/libfourcc/lib.rs

@@ -31,7 +31,7 @@ fn main() {
     let little_val = fourcc!("foo ", little);
     assert_eq!(little_val, 0x21EEFFC0u32);
 }
- ```
+```
 
 # References
 

src/libhexfloat/lib.rs

@@ -27,7 +27,7 @@ extern crate hexfloat;
 fn main() {
     let val = hexfloat!("0x1.ffffb4", f32);
 }
- ```
+```
 
 # References