Add regex matching for &[u8].

This commit enables support for compiling regular expressions that can
match on arbitrary byte slices. In particular, we add a new sub-module
called `bytes` that duplicates the API of the top-level module, except
`&str` for subjects is replaced by `&[u8]`. Additionally, Unicode
support in the regular expression is disabled by default but can be
selectively re-enabled with the `u` flag. (Unicode support cannot be
selectively disabled in the standard top-level API.)

Most of the interesting changes occurred in the `regex-syntax` crate,
where the AST now explicitly distinguishes between "ASCII compatible"
expressions and Unicode aware expressions.

This PR makes a few other changes out of convenience:

1. The DFA now knows how to "give up" if it's flushing its cache too
often. When the DFA gives up, either backtracking or the NFA algorithm
take over, which provides better performance.
2. Benchmarks were added for Oniguruma.
3. The benchmarks in general were overhauled to be defined in one place
by using conditional compilation.
4. The tests have been completely reorganized to make it easier to split
up the tests depending on which regex engine we're using. For example,
we occasionally need to be able to write tests specifically for
`regex::Regex` or specifically for `regex::bytes::Regex`.
5. Fixes a bug where NUL bytes weren't represented correctly in the byte
class optimization for the DFA.

Closes #85.

Author: BurntSushi

.travis.yml

@@ -14,10 +14,12 @@ script:
   - cargo doc --verbose --manifest-path=regex-syntax/Cargo.toml
   - if [ "$TRAVIS_RUST_VERSION" = "nightly" ]; then
       travis_wait cargo test --verbose --features pattern;
-      travis_wait cargo bench --verbose --bench dynamic;
-      travis_wait cargo bench --manifest-path=regex-pcre-benchmark/Cargo.toml --verbose
+      travis_wait ./run-bench rust;
+      travis_wait ./run-bench rust-bytes;
+      travis_wait ./run-bench rust-plugin;
+      travis_wait ./run-bench pcre;
+      travis_wait ./run-bench onig;
       travis_wait cargo test --verbose --manifest-path=regex_macros/Cargo.toml;
-      travis_wait cargo bench --manifest-path=regex_macros/Cargo.toml --verbose --bench native bench::;
     fi
 after_success: |
   [ $TRAVIS_BRANCH = master ] &&

Cargo.toml

@@ -23,10 +23,11 @@ regex-syntax = { path = "regex-syntax", version = "0.2.5" }
 utf8-ranges = "0.1"
 
 [dev-dependencies]
-# To prevent the benchmarking harness from running setup code more than once.
-# Why? Because it takes too long.
+# For examples.
 lazy_static = "0.1"
-# For generating random text to test/benchmark with.
+# For property based tests.
+quickcheck = "0.2"
+# For generating random test data.
 rand = "0.3"
 
 [features]
@@ -41,57 +42,53 @@ bench = false
 # Generally these tests specific pieces of the regex implementation.
 [[test]]
 path = "src/lib.rs"
-name = "regex"
+name = "regex-inline"
 
 # Run the test suite on the default behavior of Regex::new.
 # This includes a mish mash of NFAs and DFAs, which are chosen automatically
 # based on the regex. We test both of the NFA implementations by forcing their
 # usage with the test definitions below. (We can't test the DFA implementations
 # in the same way since they can't be used for every regex tested.)
 [[test]]
-path = "tests/test_dynamic.rs"
-name = "dynamic"
+path = "tests/test_default.rs"
+name = "default"
+test = false
 
-# Run the test suite on the NFA algorithm over Unicode codepoints.
+# The same as the default tests, but run on bytes::Regex.
 [[test]]
-path = "tests/test_dynamic_nfa.rs"
-name = "dynamic-nfa"
+path = "tests/test_default_bytes.rs"
+name = "default-bytes"
 
-# Run the test suite on the NFA algorithm over bytes.
+# Run the test suite on the NFA algorithm over Unicode codepoints.
 [[test]]
-path = "tests/test_dynamic_nfa_bytes.rs"
-name = "dynamic-nfa-bytes"
+path = "tests/test_nfa.rs"
+name = "nfa"
 
-# Run the test suite on the backtracking engine over Unicode codepoints.
+# Run the test suite on the NFA algorithm over bytes that match UTF-8 only.
 [[test]]
-path = "tests/test_dynamic_backtrack.rs"
-name = "dynamic-backtrack"
+path = "tests/test_nfa_utf8bytes.rs"
+name = "nfa-utf8bytes"
 
-# Run the test suite on the backtracking engine over bytes.
+# Run the test suite on the NFA algorithm over arbitrary bytes.
 [[test]]
-path = "tests/test_dynamic_backtrack_bytes.rs"
-name = "dynamic-backtrack-bytes"
+path = "tests/test_nfa_bytes.rs"
+name = "nfa-bytes"
 
-# Run the benchmarks on the default behavior of Regex::new.
-#
-# N.B. These benchmarks were originally taken from Russ Cox.
-[[bench]]
-name = "dynamic"
-path = "benches/bench_dynamic.rs"
-test = false
-bench = true
+# Run the test suite on the backtracking engine over Unicode codepoints.
+[[test]]
+path = "tests/test_backtrack.rs"
+name = "backtrack"
 
-# Run the benchmarks on the NFA algorithm. We avoid chasing other permutations.
-#
-# N.B. These can take a *loong* time to run.
-[[bench]]
-name = "dynamic-nfa"
-path = "benches/bench_dynamic_nfa.rs"
-test = false
-bench = true
+# Run the test suite on the backtracking engine over bytes that match UTF-8
+# only.
+[[test]]
+path = "tests/test_backtrack_utf8bytes.rs"
+name = "backtrack-utf8bytes"
 
-[profile.bench]
-debug = true
+# Run the test suite on the backtracking engine over arbitrary bytes.
+[[test]]
+path = "tests/test_backtrack_bytes.rs"
+name = "backtrack-bytes"
 
 [profile.test]
 debug = true

benches/Cargo.toml

@@ -0,0 +1,70 @@
+[package]
+publish = false
+name = "regex-benchmark"
+version = "0.1.0"
+authors = ["The Rust Project Developers"]
+license = "MIT/Apache-2.0"
+repository = "https://github.com/rust-lang/regex"
+documentation = "http://doc.rust-lang.org/regex/regex_syntax/index.html"
+homepage = "https://github.com/rust-lang/regex"
+description = "Regex benchmarks for Rust's and other engines."
+
+[dependencies]
+enum-set = "0.0.6"
+lazy_static = "0.1"
+onig = { version = "0.4", optional = true }
+pcre = { version = "0.2", optional = true }
+rand = "0.3"
+regex = { version = "0.1", path = ".." }
+regex_macros = { version = "0.1", path = "../regex_macros", optional = true }
+regex-syntax = { version = "0.2", path = "../regex-syntax" }
+
+# Use features to conditionally compile benchmarked regexes, since not every
+# regex works on every engine. :-(
+[features]
+re-pcre = ["pcre"]
+re-onig = ["onig"]
+re-rust = []
+re-rust-bytes = []
+re-rust-plugin = ["regex_macros"]
+
+# Run the benchmarks on the default behavior of Regex::new.
+[[bench]]
+name = "rust"
+path = "src/bench_rust.rs"
+test = false
+bench = true
+
+# Run the benchmarks on the default behavior of bytes::Regex::new.
+[[bench]]
+name = "rust-bytes"
+path = "src/bench_rust_bytes.rs"
+test = false
+bench = true
+
+# Run the benchmarks on the default behavior of the `regex!` compiler plugin.
+[[bench]]
+name = "rust-plugin"
+path = "src/bench_rust_plugin.rs"
+test = false
+bench = true
+
+# Run the benchmarks on PCRE.
+[[bench]]
+name = "pcre"
+path = "src/bench_pcre.rs"
+test = false
+bench = true
+
+# Run the benchmarks on Oniguruma.
+[[bench]]
+name = "onig"
+path = "src/bench_onig.rs"
+test = false
+bench = true
+
+[profile.bench]
+debug = true
+
+[profile.test]
+debug = true