Suspend the whole tokenizer and tree builder on </script>

Author: nox

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 
 name = "html5ever"
-version = "0.9.0"
+version = "0.10.0"
 authors = [ "The html5ever Project Developers" ]
 license = "MIT / Apache-2.0"
 repository = "https://github.com/servo/html5ever"

examples/noop-tokenize.rs

@@ -17,16 +17,19 @@ use std::default::Default;
 
 use tendril::{ByteTendril, ReadExt};
 
-use html5ever::tokenizer::{TokenSink, Token, Tokenizer};
+use html5ever::tokenizer::{TokenSinkResult, TokenSink, Token, Tokenizer};
 use html5ever::tokenizer::buffer_queue::BufferQueue;
 
 struct Sink(Vec<Token>);
 
 impl TokenSink for Sink {
-    fn process_token(&mut self, token: Token) {
+    type Handle = ();
+
+    fn process_token(&mut self, token: Token) -> TokenSinkResult<()> {
         // Don't use the token, but make sure we don't get
         // optimized out entirely.
         self.0.push(token);
+        TokenSinkResult::Continue
     }
 }
 

examples/tokenize.rs

@@ -15,7 +15,7 @@ use std::default::Default;
 
 use tendril::{ByteTendril, ReadExt};
 
-use html5ever::tokenizer::{TokenSink, Tokenizer, Token, TokenizerOpts, ParseError};
+use html5ever::tokenizer::{TokenSink, Tokenizer, Token, TokenizerOpts, ParseError, TokenSinkResult};
 use html5ever::tokenizer::{CharacterTokens, NullCharacterToken, TagToken, StartTag, EndTag};
 use html5ever::tokenizer::buffer_queue::BufferQueue;
 
@@ -41,7 +41,9 @@ impl TokenPrinter {
 }
 
 impl TokenSink for TokenPrinter {
-    fn process_token(&mut self, token: Token) {
+    type Handle = ();
+
+    fn process_token(&mut self, token: Token) -> TokenSinkResult<()> {
         match token {
             CharacterTokens(b) => {
                 for c in b.chars() {
@@ -74,6 +76,7 @@ impl TokenSink for TokenPrinter {
                 println!("OTHER: {:?}", token);
             }
         }
+        TokenSinkResult::Continue
     }
 }
 

src/driver.rs

@@ -86,7 +86,7 @@ pub struct Parser<Sink> where Sink: TreeSink {
 impl<Sink: TreeSink> TendrilSink<tendril::fmt::UTF8> for Parser<Sink> {
     fn process(&mut self, t: StrTendril) {
         self.input_buffer.push_front(t);
-        self.tokenizer.feed(&mut self.input_buffer)
+        let _ = self.tokenizer.feed(&mut self.input_buffer);
     }
 
     // FIXME: Is it too noisy to report every character decoding error?

src/tokenizer/interface.rs

@@ -97,16 +97,21 @@ pub enum Token {
 // FIXME: rust-lang/rust#22629
 unsafe impl Send for Token { }
 
-pub enum StateChangeQuery {
+#[derive(Debug, PartialEq)]
+#[must_use]
+pub enum TokenSinkResult<Handle> {
+    Continue,
+    Script(Handle),
     Plaintext,
-    Quiescent,
-    RawData(states::RawKind),
+    RawData(states::RawKind)
 }
 
 /// Types which can receive tokens from the tokenizer.
 pub trait TokenSink {
+    type Handle;
+
     /// Process a token.
-    fn process_token(&mut self, token: Token);
+    fn process_token(&mut self, token: Token) -> TokenSinkResult<Self::Handle>;
 
     /// Used in the markup declaration open state. By default, this always
     /// returns false and thus all CDATA sections are tokenized as bogus
@@ -115,11 +120,4 @@ pub trait TokenSink {
     fn adjusted_current_node_present_but_not_in_html_namespace(&self) -> bool {
         false
     }
-
-    /// The tokenizer will call this after emitting any tag.
-    /// This allows the tree builder to change the tokenizer's state.
-    /// By default no state changes occur.
-    fn query_state_change(&mut self) -> Option<StateChangeQuery> {
-        None
-    }
 }