From 38ce6d9eac5d0bcfa0c102bc64393a987b4a43e3 Mon Sep 17 00:00:00 2001 From: Piotr Czarnecki Date: Mon, 6 Oct 2014 23:00:56 +0100 Subject: [PATCH 1/5] Use `TokenTree`s in lhs of macros --- src/librustdoc/html/highlight.rs | 3 +- src/libsyntax/ast.rs | 50 ++++- src/libsyntax/ext/quote.rs | 52 +++-- src/libsyntax/ext/tt/macro_parser.rs | 283 ++++++++++++++++----------- src/libsyntax/ext/tt/macro_rules.rs | 45 +++-- src/libsyntax/ext/tt/transcribe.rs | 137 +++++++------ src/libsyntax/fold.rs | 13 +- src/libsyntax/parse/parser.rs | 23 ++- src/libsyntax/parse/token.rs | 8 + src/libsyntax/print/pprust.rs | 22 +-- 10 files changed, 387 insertions(+), 249 deletions(-) diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index 4797ac7c66ac6..ff50636a8d982 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -163,7 +163,8 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader, token::Lifetime(..) => "lifetime", token::DocComment(..) => "doccomment", - token::Underscore | token::Eof | token::Interpolated(..) => "", + token::Underscore | token::Eof | token::Interpolated(..) | + token::MatchNt(..) | token::SubstNt(..) => "", }; // as mentioned above, use the original source code instead of diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 078e393eb28ed..6c71f6d08d245 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -10,7 +10,7 @@ // The Rust abstract syntax tree. -use codemap::{Span, Spanned, DUMMY_SP, ExpnId}; +use codemap::{Span, Spanned, DUMMY_SP, ExpnId, respan}; use abi::Abi; use ast_util; use owned_slice::OwnedSlice; @@ -657,23 +657,55 @@ pub enum TokenTree { /// A delimited sequence of token trees TtDelimited(Span, Rc), - // These only make sense for right-hand-sides of MBE macros: + // This only makes sense for right-hand-sides of MBE macros: /// A Kleene-style repetition sequence with an optional separator. // FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST. - TtSequence(Span, Rc>, Option, KleeneOp), - /// A syntactic variable that will be filled in by macro expansion. - TtNonterminal(Span, Ident) + TtSequence(Span, Rc>, Option<::parse::token::Token>, KleeneOp, uint), } impl TokenTree { + pub fn expand_into_tts(self) -> Rc> { + match self { + TtToken(sp, token::DocComment(name)) => { + let doc = MetaNameValue(token::intern_and_get_ident("doc"), + respan(sp, LitStr(token::get_name(name), CookedStr))); + let doc = token::NtMeta(P(respan(sp, doc))); + let delimed = Delimited { + delim: token::Bracket, + open_span: sp, + tts: vec![TtToken(sp, token::Interpolated(doc))], + close_span: sp, + }; + Rc::new(vec![TtToken(sp, token::Pound), + TtDelimited(sp, Rc::new(delimed))]) + } + TtDelimited(_, ref delimed) => { + let mut tts = Vec::with_capacity(1 + delimed.tts.len() + 1); + tts.push(delimed.open_tt()); + tts.extend(delimed.tts.iter().map(|tt| tt.clone())); + tts.push(delimed.close_tt()); + Rc::new(tts) + } + TtToken(sp, token::SubstNt(name, namep)) => { + Rc::new(vec![TtToken(sp, token::Dollar), + TtToken(sp, token::Ident(name, namep))]) + } + TtToken(sp, token::MatchNt(name, kind, namep, kindp)) => { + Rc::new(vec![TtToken(sp, token::SubstNt(name, namep)), + TtToken(sp, token::Colon), + TtToken(sp, token::Ident(kind, kindp))]) + } + _ => panic!("Cannot expand a token") + } + } + /// Returns the `Span` corresponding to this token tree. pub fn get_span(&self) -> Span { match *self { - TtToken(span, _) => span, - TtDelimited(span, _) => span, - TtSequence(span, _, _, _) => span, - TtNonterminal(span, _) => span, + TtToken(span, _) => span, + TtDelimited(span, _) => span, + TtSequence(span, _, _, _, _) => span, } } } diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index f751655c9ff62..db6be89e6e9ef 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -616,6 +616,20 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P { vec!(mk_name(cx, sp, ident.ident()))); } + token::MatchNt(name, kind, name_style, kind_style) => { + return cx.expr_call(sp, + mk_token_path(cx, sp, "MatchNt"), + vec![mk_ident(cx, sp, name), + mk_ident(cx, sp, kind), + match name_style { + ModName => mk_token_path(cx, sp, "ModName"), + Plain => mk_token_path(cx, sp, "Plain"), + }, + match kind_style { + ModName => mk_token_path(cx, sp, "ModName"), + Plain => mk_token_path(cx, sp, "Plain"), + }]); + } token::Interpolated(_) => panic!("quote! with interpolated token"), _ => () @@ -654,6 +668,25 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P { fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec> { match *tt { + ast::TtToken(sp, SubstNt(ident, _)) => { + // tt.extend($ident.to_tokens(ext_cx).into_iter()) + + let e_to_toks = + cx.expr_method_call(sp, + cx.expr_ident(sp, ident), + id_ext("to_tokens"), + vec!(cx.expr_ident(sp, id_ext("ext_cx")))); + let e_to_toks = + cx.expr_method_call(sp, e_to_toks, id_ext("into_iter"), vec![]); + + let e_push = + cx.expr_method_call(sp, + cx.expr_ident(sp, id_ext("tt")), + id_ext("extend"), + vec!(e_to_toks)); + + vec!(cx.stmt_expr(e_push)) + } ast::TtToken(sp, ref tok) => { let e_sp = cx.expr_ident(sp, id_ext("_sp")); let e_tok = cx.expr_call(sp, @@ -673,25 +706,6 @@ fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec> { .collect() }, ast::TtSequence(..) => panic!("TtSequence in quote!"), - ast::TtNonterminal(sp, ident) => { - // tt.extend($ident.to_tokens(ext_cx).into_iter()) - - let e_to_toks = - cx.expr_method_call(sp, - cx.expr_ident(sp, ident), - id_ext("to_tokens"), - vec!(cx.expr_ident(sp, id_ext("ext_cx")))); - let e_to_toks = - cx.expr_method_call(sp, e_to_toks, id_ext("into_iter"), vec![]); - - let e_push = - cx.expr_method_call(sp, - cx.expr_ident(sp, id_ext("tt")), - id_ext("extend"), - vec!(e_to_toks)); - - vec!(cx.stmt_expr(e_push)) - }, } } diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index b522d5881d762..022e3a56677e2 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -78,69 +78,80 @@ use ast; -use ast::{Matcher, MatchTok, MatchSeq, MatchNonterminal, Ident}; +use ast::{Matcher, TokenTree, Ident}; +use ast::{TtDelimited, TtSequence, TtToken}; use codemap::{BytePos, mk_sp}; use codemap; use parse::lexer::*; //resolve bug? use parse::ParseSess; use parse::attr::ParserAttr; use parse::parser::{LifetimeAndTypesWithoutColons, Parser}; +use parse::token::{Eof, DocComment, MatchNt, SubstNt}; use parse::token::{Token, Nonterminal}; use parse::token; use print::pprust; use ptr::P; +use std::mem; use std::rc::Rc; use std::collections::HashMap; +use std::collections::hash_map::{Vacant, Occupied}; /* to avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body. */ +/// an unzipping of `TokenTree`s +#[deriving(Clone)] +struct MatcherTtFrame { + elts: Rc>, + idx: uint, +} + #[deriving(Clone)] pub struct MatcherPos { - elts: Vec , // maybe should be <'>? Need to understand regions. + stack: Vec, + elts: Rc>, sep: Option, idx: uint, up: Option>, matches: Vec>>, - match_lo: uint, match_hi: uint, + match_lo: uint, + match_cur: uint, + match_hi: uint, sp_lo: BytePos, } -pub fn count_names(ms: &[Matcher]) -> uint { - ms.iter().fold(0, |ct, m| { - ct + match m.node { - MatchTok(_) => 0u, - MatchSeq(ref more_ms, _, _, _, _) => { - count_names(more_ms.as_slice()) +pub fn count_names(ms: &[TokenTree]) -> uint { + ms.iter().fold(0, |count, elt| { + count + match elt { + &TtSequence(_, _, _, _, advance_by) => { + advance_by } - MatchNonterminal(_, _, _) => 1u - }}) -} - -pub fn initial_matcher_pos(ms: Vec , sep: Option, lo: BytePos) - -> Box { - let mut match_idx_hi = 0u; - for elt in ms.iter() { - match elt.node { - MatchTok(_) => (), - MatchSeq(_,_,_,_,hi) => { - match_idx_hi = hi; // it is monotonic... + &TtDelimited(_, ref delim) => { + count_names(delim.tts.as_slice()) } - MatchNonterminal(_,_,pos) => { - match_idx_hi = pos+1u; // ...so latest is highest + &TtToken(_, MatchNt(..)) => { + 1 } + &TtToken(_, _) => 0, } - } - let matches = Vec::from_fn(count_names(ms.as_slice()), |_i| Vec::new()); + }) +} + +pub fn initial_matcher_pos(ms: Rc>, sep: Option, lo: BytePos) + -> Box { + let match_idx_hi = count_names(ms.as_slice()); + let matches = Vec::from_fn(match_idx_hi, |_i| Vec::new()); box MatcherPos { + stack: vec![], elts: ms, sep: sep, idx: 0u, up: None, matches: matches, match_lo: 0u, + match_cur: 0u, match_hi: match_idx_hi, sp_lo: lo } @@ -152,11 +163,9 @@ pub fn initial_matcher_pos(ms: Vec , sep: Option, lo: BytePos) /// (expr, item, etc). All the leaves in a single NamedMatch correspond to a /// single matcher_nonterminal in the ast::Matcher that produced it. /// -/// It should probably be renamed, it has more or less exact correspondence to -/// ast::match nodes, and the in-memory structure of a particular NamedMatch -/// represents the match that occurred when a particular subset of an -/// ast::match -- those ast::Matcher nodes leading to a single -/// MatchNonterminal -- was applied to a particular token tree. +/// The in-memory structure of a particular NamedMatch represents the match +/// that occurred when a particular subset of a matcher was applied to a +/// particular token tree. /// /// The width of each MatchedSeq in the NamedMatch, and the identity of the /// MatchedNonterminal's, will depend on the token tree it was applied to: each @@ -170,34 +179,43 @@ pub enum NamedMatch { MatchedNonterminal(Nonterminal) } -pub fn nameize(p_s: &ParseSess, ms: &[Matcher], res: &[Rc]) +pub fn nameize(p_s: &ParseSess, ms: &[TokenTree], res: &[Rc]) -> HashMap> { - fn n_rec(p_s: &ParseSess, m: &Matcher, res: &[Rc], - ret_val: &mut HashMap>) { - match *m { - codemap::Spanned {node: MatchTok(_), .. } => (), - codemap::Spanned {node: MatchSeq(ref more_ms, _, _, _, _), .. } => { - for next_m in more_ms.iter() { - n_rec(p_s, next_m, res, ret_val) - }; - } - codemap::Spanned { - node: MatchNonterminal(bind_name, _, idx), - span - } => { - if ret_val.contains_key(&bind_name) { - let string = token::get_ident(bind_name); - p_s.span_diagnostic - .span_fatal(span, - format!("duplicated bind name: {}", - string.get()).as_slice()) + fn n_rec(p_s: &ParseSess, m: &TokenTree, res: &[Rc], + ret_val: &mut HashMap>, idx: &mut uint) { + match m { + &TtSequence(_, ref more_ms, _, _, _) => { + for next_m in more_ms.iter() { + n_rec(p_s, next_m, res, ret_val, idx) + } + } + &TtDelimited(_, ref delim) => { + for next_m in delim.tts.iter() { + n_rec(p_s, next_m, res, ret_val, idx) + } + } + &TtToken(sp, MatchNt(bind_name, _, _, _)) => { + match ret_val.entry(bind_name) { + Vacant(spot) => { + spot.set(res[*idx].clone()); + *idx += 1; + } + Occupied(..) => { + let string = token::get_ident(bind_name); + p_s.span_diagnostic + .span_fatal(sp, + format!("duplicated bind name: {}", + string.get()).as_slice()) + } + } } - ret_val.insert(bind_name, res[idx].clone()); - } + &TtToken(_, SubstNt(..)) => panic!("Cannot fill in a NT"), + &TtToken(_, _) => (), } } let mut ret_val = HashMap::new(); - for m in ms.iter() { n_rec(p_s, m, res, &mut ret_val) } + let mut idx = 0u; + for m in ms.iter() { n_rec(p_s, m, res, &mut ret_val, &mut idx) } ret_val } @@ -210,7 +228,7 @@ pub enum ParseResult { pub fn parse_or_else(sess: &ParseSess, cfg: ast::CrateConfig, rdr: TtReader, - ms: Vec ) + ms: Vec ) -> HashMap> { match parse(sess, cfg, rdr, ms.as_slice()) { Success(m) => m, @@ -237,12 +255,12 @@ pub fn token_name_eq(t1 : &Token, t2 : &Token) -> bool { pub fn parse(sess: &ParseSess, cfg: ast::CrateConfig, mut rdr: TtReader, - ms: &[Matcher]) + ms: &[TokenTree]) -> ParseResult { let mut cur_eis = Vec::new(); - cur_eis.push(initial_matcher_pos(ms.iter() - .map(|x| (*x).clone()) - .collect(), + cur_eis.push(initial_matcher_pos(Rc::new(ms.iter() + .map(|x| (*x).clone()) + .collect()), None, rdr.peek().sp.lo)); @@ -255,11 +273,22 @@ pub fn parse(sess: &ParseSess, /* we append new items to this while we go */ loop { - let ei = match cur_eis.pop() { + let mut ei = match cur_eis.pop() { None => break, /* for each Earley Item */ Some(ei) => ei, }; + // When unzipped trees end, remove them + while ei.idx >= ei.elts.len() { + match ei.stack.pop() { + Some(MatcherTtFrame { elts, idx }) => { + ei.elts = elts; + ei.idx = idx + 1; + } + None => break + } + } + let idx = ei.idx; let len = ei.elts.len(); @@ -293,6 +322,7 @@ pub fn parse(sess: &ParseSess, sp.hi)))); } + new_pos.match_cur = ei.match_hi; new_pos.idx += 1; cur_eis.push(new_pos); } @@ -301,69 +331,88 @@ pub fn parse(sess: &ParseSess, // the *_t vars are workarounds for the lack of unary move match ei.sep { - Some(ref t) if idx == len => { // we need a separator - // i'm conflicted about whether this should be hygienic.... - // though in this case, if the separators are never legal - // idents, it shouldn't matter. - if token_name_eq(&tok, t) { //pass the separator - let mut ei_t = ei.clone(); - ei_t.idx += 1; - next_eis.push(ei_t); + Some(ref t) if idx == len => { // we need a separator + // i'm conflicted about whether this should be hygienic.... + // though in this case, if the separators are never legal + // idents, it shouldn't matter. + if token_name_eq(&tok, t) { //pass the separator + let mut ei_t = ei.clone(); + // ei_t.match_cur = ei_t.match_lo; + ei_t.idx += 1; + next_eis.push(ei_t); + } + } + _ => { // we don't need a separator + let mut ei_t = ei; + ei_t.match_cur = ei_t.match_lo; + ei_t.idx = 0; + cur_eis.push(ei_t); } - } - _ => { // we don't need a separator - let mut ei_t = ei; - ei_t.idx = 0; - cur_eis.push(ei_t); - } } } else { eof_eis.push(ei); } } else { - match ei.elts[idx].node.clone() { - /* need to descend into sequence */ - MatchSeq(ref matchers, ref sep, kleene_op, - match_idx_lo, match_idx_hi) => { - if kleene_op == ast::ZeroOrMore { - let mut new_ei = ei.clone(); - new_ei.idx += 1u; - //we specifically matched zero repeats. - for idx in range(match_idx_lo, match_idx_hi) { - new_ei.matches[idx] - .push(Rc::new(MatchedSeq(Vec::new(), sp))); + match (*ei.elts)[idx].clone() { + /* need to descend into sequence */ + TtSequence(_, ref matchers, ref sep, kleene_op, match_num) => { + if kleene_op == ast::ZeroOrMore { + let mut new_ei = ei.clone(); + new_ei.match_cur += match_num; + new_ei.idx += 1u; + //we specifically matched zero repeats. + for idx in range(ei.match_cur, ei.match_cur + match_num) { + new_ei.matches[idx] + .push(Rc::new(MatchedSeq(Vec::new(), sp))); + } + + cur_eis.push(new_ei); } - cur_eis.push(new_ei); + let matches = Vec::from_elem(ei.matches.len(), Vec::new()); + let ei_t = ei; + cur_eis.push(box MatcherPos { + stack: vec![], + elts: matchers.clone(), + sep: (*sep).clone(), + idx: 0u, + matches: matches, + match_lo: ei_t.match_cur, + match_cur: ei_t.match_cur, + match_hi: ei_t.match_cur + match_num, + up: Some(ei_t), + sp_lo: sp.lo + }); } - - let matches = Vec::from_elem(ei.matches.len(), Vec::new()); - let ei_t = ei; - cur_eis.push(box MatcherPos { - elts: (*matchers).clone(), - sep: (*sep).clone(), - idx: 0u, - up: Some(ei_t), - matches: matches, - match_lo: match_idx_lo, match_hi: match_idx_hi, - sp_lo: sp.lo - }); - } - MatchNonterminal(_,_,_) => { - // Built-in nonterminals never start with these tokens, - // so we can eliminate them from consideration. - match tok { - token::CloseDelim(_) => {}, - _ => bb_eis.push(ei), + TtToken(_, MatchNt(..)) => { + // Built-in nonterminals never start with these tokens, + // so we can eliminate them from consideration. + match tok { + token::CloseDelim(_) => {}, + _ => bb_eis.push(ei), + } } - } - MatchTok(ref t) => { - let mut ei_t = ei.clone(); - if token_name_eq(t,&tok) { - ei_t.idx += 1; - next_eis.push(ei_t); + TtToken(sp, SubstNt(..)) => { + return Error(sp, "Cannot transcribe in macro LHS".into_string()) + } + seq @ TtDelimited(..) | seq @ TtToken(_, DocComment(..)) => { + let tts = seq.expand_into_tts(); + let elts = mem::replace(&mut ei.elts, tts); + let idx = ei.idx; + ei.stack.push(MatcherTtFrame { + elts: elts, + idx: idx, + }); + ei.idx = 0; + cur_eis.push(ei); + } + TtToken(_, ref t) => { + let mut ei_t = ei.clone(); + if token_name_eq(t,&tok) { + ei_t.idx += 1; + next_eis.push(ei_t); + } } - } } } } @@ -385,8 +434,8 @@ pub fn parse(sess: &ParseSess, if (bb_eis.len() > 0u && next_eis.len() > 0u) || bb_eis.len() > 1u { let nts = bb_eis.iter().map(|ei| { - match ei.elts[ei.idx].node { - MatchNonterminal(bind, name, _) => { + match (*ei.elts)[ei.idx] { + TtToken(_, MatchNt(bind, name, _, _)) => { (format!("{} ('{}')", token::get_ident(name), token::get_ident(bind))).to_string() @@ -410,12 +459,14 @@ pub fn parse(sess: &ParseSess, let mut rust_parser = Parser::new(sess, cfg.clone(), box rdr.clone()); let mut ei = bb_eis.pop().unwrap(); - match ei.elts[ei.idx].node { - MatchNonterminal(_, name, idx) => { + match (*ei.elts)[ei.idx] { + TtToken(_, MatchNt(_, name, _, _)) => { let name_string = token::get_ident(name); - ei.matches[idx].push(Rc::new(MatchedNonterminal( + let match_cur = ei.match_cur; + ei.matches[match_cur].push(Rc::new(MatchedNonterminal( parse_nt(&mut rust_parser, name_string.get())))); ei.idx += 1u; + ei.match_cur += 1; } _ => panic!() } diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index e50d4457af249..cbf34ba5eb320 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -9,8 +9,9 @@ // except according to those terms. use ast::{Ident, Matcher_, Matcher, MatchTok, MatchNonterminal, MatchSeq, TtDelimited}; +use ast::{TtSequence, TtToken}; use ast; -use codemap::{Span, Spanned, DUMMY_SP}; +use codemap::{Span, DUMMY_SP}; use ext::base::{ExtCtxt, MacResult, MacroDef}; use ext::base::{NormalTT, TTMacroExpander}; use ext::tt::macro_parser::{Success, Error, Failure}; @@ -20,7 +21,7 @@ use parse::lexer::new_tt_reader; use parse::parser::Parser; use parse::attr::ParserAttr; use parse::token::{special_idents, gensym_ident}; -use parse::token::{NtMatchers, NtTT}; +use parse::token::{MatchNt, NtMatchers, NtTT}; use parse::token; use print; use ptr::P; @@ -158,14 +159,18 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, for (i, lhs) in lhses.iter().enumerate() { // try each arm's matchers match **lhs { - MatchedNonterminal(NtMatchers(ref mtcs)) => { + MatchedNonterminal(NtTT(ref lhs_tt)) => { + let lhs_tt = match **lhs_tt { + TtDelimited(_, ref delim) => delim.tts.as_slice(), + _ => cx.span_fatal(sp, "malformed macro lhs") + }; // `None` is because we're not interpolating let arg_rdr = new_tt_reader(&cx.parse_sess().span_diagnostic, None, arg.iter() .map(|x| (*x).clone()) .collect()); - match parse(cx.parse_sess(), cx.cfg(), arg_rdr, mtcs.as_slice()) { + match parse(cx.parse_sess(), cx.cfg(), arg_rdr, lhs_tt) { Success(named_matches) => { let rhs = match *rhses[i] { // okay, what's your transcriber? @@ -210,31 +215,33 @@ pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt, name: Ident, arg: Vec ) -> Box { - // these spans won't matter, anyways - fn ms(m: Matcher_) -> Matcher { - Spanned { - node: m.clone(), - span: DUMMY_SP - } - } let lhs_nm = gensym_ident("lhs"); let rhs_nm = gensym_ident("rhs"); // The pattern that macro_rules matches. // The grammar for macro_rules! is: - // $( $lhs:mtcs => $rhs:tt );+ + // $( $lhs:tt => $rhs:tt );+ // ...quasiquoting this would be nice. + // These spans won't matter, anyways + let match_lhs_tok = MatchNt(lhs_nm, special_idents::tt, token::Plain, token::Plain); + let match_rhs_tok = MatchNt(rhs_nm, special_idents::tt, token::Plain, token::Plain); let argument_gram = vec!( - ms(MatchSeq(vec!( - ms(MatchNonterminal(lhs_nm, special_idents::matchers, 0u)), - ms(MatchTok(token::FatArrow)), - ms(MatchNonterminal(rhs_nm, special_idents::tt, 1u))), - Some(token::Semi), ast::OneOrMore, 0u, 2u)), + TtSequence(DUMMY_SP, + Rc::new(vec![ + TtToken(DUMMY_SP, match_lhs), + TtToken(DUMMY_SP, token::FatArrow), + TtToken(DUMMY_SP, match_rhs)]), + Some(token::Semi), + ast::OneOrMore, + 2), //to phase into semicolon-termination instead of //semicolon-separation - ms(MatchSeq(vec!(ms(MatchTok(token::Semi))), None, - ast::ZeroOrMore, 2u, 2u))); + TtSequence(DUMMY_SP, + Rc::new(vec![TtToken(DUMMY_SP, token::Semi)]), + None, + ast::ZeroOrMore, + 0)); // Parse the macro_rules! invocation (`none` is for no interpolations): diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 249a985a6488a..e8a2d9a243362 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -9,10 +9,11 @@ // except according to those terms. use ast; -use ast::{TokenTree, TtDelimited, TtToken, TtSequence, TtNonterminal, Ident}; +use ast::{TokenTree, TtDelimited, TtToken, TtSequence, Ident}; use codemap::{Span, DUMMY_SP}; use diagnostic::SpanHandler; use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal}; +use parse::token::{Eof, DocComment, Interpolated, MatchNt, SubstNt}; use parse::token::{Token, NtIdent}; use parse::token; use parse::lexer::TokenAndSpan; @@ -85,17 +86,9 @@ fn lookup_cur_matched_by_matched(r: &TtReader, start: Rc) -> Rc Rc { +fn lookup_cur_matched(r: &TtReader, name: Ident) -> Option> { let matched_opt = r.interpolations.find_copy(&name); - match matched_opt { - Some(s) => lookup_cur_matched_by_matched(r, s), - None => { - r.sp_diag - .span_fatal(r.cur_span, - format!("unknown macro variable `{}`", - token::get_ident(name)).as_slice()); - } - } + matched_opt.map(|s| lookup_cur_matched_by_matched(r, s)) } #[deriving(Clone)] @@ -133,16 +126,20 @@ fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize { size + lockstep_iter_size(tt, r) }) }, - TtSequence(_, ref tts, _, _) => { + TtSequence(_, ref tts, _, _, _) => { tts.iter().fold(LisUnconstrained, |size, tt| { size + lockstep_iter_size(tt, r) }) }, + TtToken(_, SubstNt(name, _)) | TtToken(_, MatchNt(name, _, _, _)) => + match lookup_cur_matched(r, name) { + Some(matched) => match *matched { + MatchedNonterminal(_) => LisUnconstrained, + MatchedSeq(ref ads, _) => LisConstraint(ads.len(), name), + }, + _ => LisUnconstrained + }, TtToken(..) => LisUnconstrained, - TtNonterminal(_, name) => match *lookup_cur_matched(r, name) { - MatchedNonterminal(_) => LisUnconstrained, - MatchedSeq(ref ads, _) => LisConstraint(ads.len(), name) - }, } } @@ -205,40 +202,21 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { (*frame.forest)[frame.idx].clone() }; match t { - TtDelimited(_, ref delimed) => { - let mut tts = Vec::with_capacity(1 + delimed.tts.len() + 1); - tts.push(delimed.open_tt()); - tts.extend(delimed.tts.iter().map(|tt| tt.clone())); - tts.push(delimed.close_tt()); - - r.stack.push(TtFrame { - forest: Rc::new(tts), - idx: 0, - dotdotdoted: false, - sep: None - }); - // if this could be 0-length, we'd need to potentially recur here - } - TtToken(sp, tok) => { - r.cur_span = sp; - r.cur_tok = tok; - r.stack.last_mut().unwrap().idx += 1; - return ret_val; - } - TtSequence(sp, tts, sep, kleene_op) => { + TtSequence(sp, tts, sep, kleene_op, n) => { // FIXME(pcwalton): Bad copy. - match lockstep_iter_size(&TtSequence(sp, tts.clone(), sep.clone(), kleene_op), r) { + match lockstep_iter_size(&TtSequence(sp, tts.clone(), sep.clone(), kleene_op, n), + r) { LisUnconstrained => { r.sp_diag.span_fatal( sp.clone(), /* blame macro writer */ "attempted to repeat an expression \ containing no syntax \ variables matched as repeating at this depth"); - } - LisContradiction(ref msg) => { - // FIXME #2887 blame macro invoker instead - r.sp_diag.span_fatal(sp.clone(), msg.as_slice()); - } + } + LisContradiction(ref msg) => { + // FIXME #2887 blame macro invoker instead + r.sp_diag.span_fatal(sp.clone(), msg.as_slice()); + } LisConstraint(len, _) => { if len == 0 { if kleene_op == ast::OneOrMore { @@ -262,31 +240,62 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { } } // FIXME #2887: think about span stuff here - TtNonterminal(sp, ident) => { - r.stack.last_mut().unwrap().idx += 1; - match *lookup_cur_matched(r, ident) { - /* sidestep the interpolation tricks for ident because - (a) idents can be in lots of places, so it'd be a pain - (b) we actually can, since it's a token. */ - MatchedNonterminal(NtIdent(box sn, b)) => { - r.cur_span = sp; - r.cur_tok = token::Ident(sn,b); - return ret_val; - } - MatchedNonterminal(ref other_whole_nt) => { - // FIXME(pcwalton): Bad copy. - r.cur_span = sp; - r.cur_tok = token::Interpolated((*other_whole_nt).clone()); - return ret_val; + TtToken(sp, SubstNt(ident, namep)) => { + match lookup_cur_matched(r, ident) { + None => { + r.stack.push(TtFrame { + forest: TtToken(sp, SubstNt(ident, namep)).expand_into_tts(), + idx: 0, + dotdotdoted: false, + sep: None + }); + // this can't be 0 length, just like TtDelimited } - MatchedSeq(..) => { - r.sp_diag.span_fatal( - r.cur_span, /* blame the macro writer */ - format!("variable '{}' is still repeating at this depth", - token::get_ident(ident)).as_slice()); + Some(cur_matched) => { + r.stack.last_mut().unwrap().idx += 1; + match *cur_matched { + // sidestep the interpolation tricks for ident because + // (a) idents can be in lots of places, so it'd be a pain + // (b) we actually can, since it's a token. + MatchedNonterminal(NtIdent(box sn, b)) => { + r.cur_span = sp; + r.cur_tok = token::Ident(sn, b); + return ret_val; + } + MatchedNonterminal(ref other_whole_nt) => { + // FIXME(pcwalton): Bad copy. + r.cur_span = sp; + r.cur_tok = token::Interpolated((*other_whole_nt).clone()); + return ret_val; + } + MatchedSeq(..) => { + r.sp_diag.span_fatal( + r.cur_span, /* blame the macro writer */ + format!("variable '{}' is still repeating at this depth", + token::get_ident(ident)).as_slice()); + } + } } } } + // TtDelimited or any token that can be unzipped + seq @ TtDelimited(..) | seq @ TtToken(_, DocComment(..)) + | seq @ TtToken(_, MatchNt(..)) => { + // do not advance the idx yet + r.stack.push(TtFrame { + forest: seq.expand_into_tts(), + idx: 0, + dotdotdoted: false, + sep: None + }); + // if this could be 0-length, we'd need to potentially recur here + } + TtToken(sp, tok) => { + r.cur_span = sp; + r.cur_tok = tok; + r.stack.last_mut().unwrap().idx += 1; + return ret_val; + } } } } diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 6535c8e89fd4e..75fbdb8d8b6b0 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -581,13 +581,12 @@ pub fn noop_fold_tt(tt: &TokenTree, fld: &mut T) -> TokenTree { } )) }, - TtSequence(span, ref pattern, ref sep, is_optional) => + TtSequence(span, ref pattern, ref sep, is_optional, advance_by) => TtSequence(span, Rc::new(fld.fold_tts(pattern.as_slice())), sep.clone().map(|tok| fld.fold_token(tok)), - is_optional), - TtNonterminal(sp,ref ident) => - TtNonterminal(sp,fld.fold_ident(*ident)) + is_optional, + advance_by), } } @@ -603,6 +602,12 @@ pub fn noop_fold_token(t: token::Token, fld: &mut T) -> token::Token } token::Lifetime(id) => token::Lifetime(fld.fold_ident(id)), token::Interpolated(nt) => token::Interpolated(fld.fold_interpolated(nt)), + token::SubstNt(ident, namep) => { + token::SubstNt(fld.fold_ident(ident), namep) + } + token::MatchNt(name, kind, namep, kindp) => { + token::MatchNt(fld.fold_ident(name), fld.fold_ident(kind), namep, kindp) + } _ => t } } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 10bb9ef36250e..741014fec8962 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -49,7 +49,7 @@ use ast::{StructVariantKind, BiSub}; use ast::StrStyle; use ast::{SelfExplicit, SelfRegion, SelfStatic, SelfValue}; use ast::{Delimited, TokenTree, TraitItem, TraitRef, TtDelimited, TtSequence, TtToken}; -use ast::{TtNonterminal, TupleVariantKind, Ty, Ty_, TyBot}; +use ast::{TupleVariantKind, Ty, Ty_, TyBot}; use ast::{TypeField, TyFixedLengthVec, TyClosure, TyProc, TyBareFn}; use ast::{TyTypeof, TyInfer, TypeMethod}; use ast::{TyNil, TyParam, TyParamBound, TyParen, TyPath, TyPtr, TyQPath}; @@ -65,6 +65,7 @@ use ast_util::{as_prec, ident_to_path, operator_prec}; use ast_util; use codemap::{Span, BytePos, Spanned, spanned, mk_sp}; use codemap; +use ext::tt::macro_parser; use parse; use parse::attr::ParserAttr; use parse::classify; @@ -73,7 +74,7 @@ use parse::common::{seq_sep_trailing_allowed}; use parse::lexer::Reader; use parse::lexer::TokenAndSpan; use parse::obsolete::*; -use parse::token::InternedString; +use parse::token::{MatchNt, SubstNt, InternedString}; use parse::token::{keywords, special_idents}; use parse::token; use parse::{new_sub_parser_from_file, ParseSess}; @@ -2508,7 +2509,7 @@ impl<'a> Parser<'a> { pub fn parse_token_tree(&mut self) -> TokenTree { // FIXME #6994: currently, this is too eager. It // parses token trees but also identifies TtSequence's - // and TtNonterminal's; it's too early to know yet + // and token::SubstNt's; it's too early to know yet // whether something will be a nonterminal or a seq // yet. maybe_whole!(deref self, NtTT); @@ -2549,9 +2550,21 @@ impl<'a> Parser<'a> { let seq = match seq { Spanned { node, .. } => node, }; - TtSequence(mk_sp(sp.lo, p.span.hi), Rc::new(seq), sep, repeat) + let name_num = macro_parser::count_names(seq.as_slice()); + TtSequence(mk_sp(sp.lo, p.span.hi), Rc::new(seq), sep, repeat, name_num) } else { - TtNonterminal(sp, p.parse_ident()) + // A nonterminal that matches or not + let namep = match p.token { token::Ident(_, p) => p, _ => token::Plain }; + let name = p.parse_ident(); + if p.token == token::Colon && p.look_ahead(1, |t| t.is_ident()) { + p.bump(); + let kindp = match p.token { token::Ident(_, p) => p, _ => token::Plain }; + let nt_kind = p.parse_ident(); + let m = TtToken(sp, MatchNt(name, nt_kind, namep, kindp)); + m + } else { + TtToken(sp, SubstNt(name, namep)) + } } } _ => { diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 29a2f38959eed..8dd2f8b840f92 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -108,7 +108,15 @@ pub enum Token { /* For interpolation */ Interpolated(Nonterminal), + // Can be expanded into several tokens. + /// Doc comment DocComment(ast::Name), + // In left-hand-sides of MBE macros: + /// Parse a nonterminal (name to bind, name of NT, styles of their idents) + MatchNt(ast::Ident, ast::Ident, IdentStyle, IdentStyle), + // In right-hand-sides of MBE macros: + /// A syntactic variable that will be filled in by macro expansion. + SubstNt(ast::Ident, IdentStyle), // Junk. These carry no data because we don't really care about the data // they *would* carry, and don't really want to allocate a new ident for diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 106e3f1faae89..12ce81eedca8c 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -254,6 +254,8 @@ pub fn token_to_string(tok: &Token) -> String { /* Other */ token::DocComment(s) => s.as_str().into_string(), + token::SubstNt(s, _) => format!("${}", s), + token::MatchNt(s, t, _, _) => format!("${}:{}", s, t), token::Eof => "".into_string(), token::Whitespace => " ".into_string(), token::Comment => "/* */".into_string(), @@ -1120,13 +1122,6 @@ impl<'a> State<'a> { /// expression arguments as expressions). It can be done! I think. pub fn print_tt(&mut self, tt: &ast::TokenTree) -> IoResult<()> { match *tt { - ast::TtDelimited(_, ref delimed) => { - try!(word(&mut self.s, token_to_string(&delimed.open_token()).as_slice())); - try!(space(&mut self.s)); - try!(self.print_tts(delimed.tts.as_slice())); - try!(space(&mut self.s)); - word(&mut self.s, token_to_string(&delimed.close_token()).as_slice()) - }, ast::TtToken(_, ref tk) => { try!(word(&mut self.s, token_to_string(tk).as_slice())); match *tk { @@ -1136,7 +1131,14 @@ impl<'a> State<'a> { _ => Ok(()) } } - ast::TtSequence(_, ref tts, ref separator, kleene_op) => { + ast::TtDelimited(_, ref delimed) => { + try!(word(&mut self.s, token_to_string(&delimed.open_token()).as_slice())); + try!(space(&mut self.s)); + try!(self.print_tts(delimed.tts.as_slice())); + try!(space(&mut self.s)); + word(&mut self.s, token_to_string(&delimed.close_token()).as_slice()) + }, + ast::TtSequence(_, ref tts, ref separator, kleene_op, _) => { try!(word(&mut self.s, "$(")); for tt_elt in (*tts).iter() { try!(self.print_tt(tt_elt)); @@ -1153,10 +1155,6 @@ impl<'a> State<'a> { ast::OneOrMore => word(&mut self.s, "+"), } } - ast::TtNonterminal(_, name) => { - try!(word(&mut self.s, "$")); - self.print_ident(name) - } } } From 6f30a4ee6c35342cc2775d77882ad26fc31ba61e Mon Sep 17 00:00:00 2001 From: Piotr Czarnecki Date: Tue, 7 Oct 2014 00:18:24 +0100 Subject: [PATCH 2/5] Remove `Matcher`s --- src/libsyntax/ast.rs | 84 ++++------------------------ src/libsyntax/ext/tt/macro_parser.rs | 26 ++++----- src/libsyntax/ext/tt/macro_rules.rs | 10 +++- src/libsyntax/fold.rs | 2 - src/libsyntax/parse/parser.rs | 64 +-------------------- src/libsyntax/parse/token.rs | 2 - src/libsyntax/print/pprust.rs | 1 - 7 files changed, 33 insertions(+), 156 deletions(-) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 6c71f6d08d245..639b007e7ab24 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -641,14 +641,12 @@ pub enum KleeneOp { /// be passed to syntax extensions using a uniform type. /// /// If the syntax extension is an MBE macro, it will attempt to match its -/// LHS "matchers" against the provided token tree, and if it finds a +/// LHS token tree against the provided token tree, and if it finds a /// match, will transcribe the RHS token tree, splicing in any captured -/// `macro_parser::matched_nonterminals` into the `TtNonterminal`s it finds. +/// macro_parser::matched_nonterminals into the `SubstNt`s it finds. /// -/// The RHS of an MBE macro is the only place a `TtNonterminal` or `TtSequence` -/// makes any real sense. You could write them elsewhere but nothing -/// else knows what to do with them, so you'll probably get a syntax -/// error. +/// The RHS of an MBE macro is the only place `SubstNt`s are substituted. +/// Nothing special happens to misnamed or misplaced `SubstNt`s. #[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)] #[doc="For macro invocations; parsing is delegated to the macro"] pub enum TokenTree { @@ -657,14 +655,19 @@ pub enum TokenTree { /// A delimited sequence of token trees TtDelimited(Span, Rc), - // This only makes sense for right-hand-sides of MBE macros: + // This only makes sense in MBE macros. - /// A Kleene-style repetition sequence with an optional separator. - // FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST. + /// A kleene-style repetition sequence with a span, a TT forest, + /// an optional separator, and a boolean where true indicates + /// zero or more (..), and false indicates one or more (+). + /// The last member denotes the number of `MATCH_NONTERMINAL`s + /// in the forest. + // FIXME(eddyb) #12938 Use Rc<[TokenTree]> after DST. TtSequence(Span, Rc>, Option<::parse::token::Token>, KleeneOp, uint), } impl TokenTree { + /// For unrolling some tokens or token trees into equivalent sequences. pub fn expand_into_tts(self) -> Rc> { match self { TtToken(sp, token::DocComment(name)) => { @@ -710,69 +713,6 @@ impl TokenTree { } } -// Matchers are nodes defined-by and recognized-by the main rust parser and -// language, but they're only ever found inside syntax-extension invocations; -// indeed, the only thing that ever _activates_ the rules in the rust parser -// for parsing a matcher is a matcher looking for the 'matchers' nonterminal -// itself. Matchers represent a small sub-language for pattern-matching -// token-trees, and are thus primarily used by the macro-defining extension -// itself. -// -// MatchTok -// -------- -// -// A matcher that matches a single token, denoted by the token itself. So -// long as there's no $ involved. -// -// -// MatchSeq -// -------- -// -// A matcher that matches a sequence of sub-matchers, denoted various -// possible ways: -// -// $(M)* zero or more Ms -// $(M)+ one or more Ms -// $(M),+ one or more comma-separated Ms -// $(A B C);* zero or more semi-separated 'A B C' seqs -// -// -// MatchNonterminal -// ----------------- -// -// A matcher that matches one of a few interesting named rust -// nonterminals, such as types, expressions, items, or raw token-trees. A -// black-box matcher on expr, for example, binds an expr to a given ident, -// and that ident can re-occur as an interpolation in the RHS of a -// macro-by-example rule. For example: -// -// $foo:expr => 1 + $foo // interpolate an expr -// $foo:tt => $foo // interpolate a token-tree -// $foo:tt => bar! $foo // only other valid interpolation -// // is in arg position for another -// // macro -// -// As a final, horrifying aside, note that macro-by-example's input is -// also matched by one of these matchers. Holy self-referential! It is matched -// by a MatchSeq, specifically this one: -// -// $( $lhs:matchers => $rhs:tt );+ -// -// If you understand that, you have closed the loop and understand the whole -// macro system. Congratulations. -pub type Matcher = Spanned; - -#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)] -pub enum Matcher_ { - /// Match one token - MatchTok(token::Token), - /// Match repetitions of a sequence: body, separator, Kleene operator, - /// lo, hi position-in-match-array used: - MatchSeq(Vec, Option, KleeneOp, uint, uint), - /// Parse a Rust NT: name to bind, name of NT, position in match array: - MatchNonterminal(Ident, Ident, uint) -} - pub type Mac = Spanned; /// Represents a macro invocation. The Path indicates which macro diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index 022e3a56677e2..833211f53e7aa 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -78,7 +78,7 @@ use ast; -use ast::{Matcher, TokenTree, Ident}; +use ast::{TokenTree, Ident}; use ast::{TtDelimited, TtSequence, TtToken}; use codemap::{BytePos, mk_sp}; use codemap; @@ -97,9 +97,8 @@ use std::rc::Rc; use std::collections::HashMap; use std::collections::hash_map::{Vacant, Occupied}; -/* to avoid costly uniqueness checks, we require that `MatchSeq` always has a -nonempty body. */ - +// To avoid costly uniqueness checks, we require that `MatchSeq` always has +// a nonempty body. /// an unzipping of `TokenTree`s #[deriving(Clone)] @@ -157,22 +156,22 @@ pub fn initial_matcher_pos(ms: Rc>, sep: Option, lo: ByteP } } -/// NamedMatch is a pattern-match result for a single ast::MatchNonterminal: +/// NamedMatch is a pattern-match result for a single token::MATCH_NONTERMINAL: /// so it is associated with a single ident in a parse, and all -/// MatchedNonterminal's in the NamedMatch have the same nonterminal type -/// (expr, item, etc). All the leaves in a single NamedMatch correspond to a -/// single matcher_nonterminal in the ast::Matcher that produced it. +/// `MatchedNonterminal`s in the NamedMatch have the same nonterminal type +/// (expr, item, etc). Each leaf in a single NamedMatch corresponds to a +/// single token::MATCH_NONTERMINAL in the TokenTree that produced it. /// /// The in-memory structure of a particular NamedMatch represents the match /// that occurred when a particular subset of a matcher was applied to a /// particular token tree. /// /// The width of each MatchedSeq in the NamedMatch, and the identity of the -/// MatchedNonterminal's, will depend on the token tree it was applied to: each -/// MatchedSeq corresponds to a single MatchSeq in the originating -/// ast::Matcher. The depth of the NamedMatch structure will therefore depend -/// only on the nesting depth of ast::MatchSeq's in the originating -/// ast::Matcher it was derived from. +/// `MatchedNonterminal`s, will depend on the token tree it was applied to: +/// each MatchedSeq corresponds to a single TTSeq in the originating +/// token tree. The depth of the NamedMatch structure will therefore depend +/// only on the nesting depth of `ast::TTSeq`s in the originating +/// token tree it was derived from. pub enum NamedMatch { MatchedSeq(Vec>, codemap::Span), @@ -512,7 +511,6 @@ pub fn parse_nt(p: &mut Parser, name: &str) -> Nonterminal { p.quote_depth -= 1u; res } - "matchers" => token::NtMatchers(p.parse_matchers()), _ => { p.fatal(format!("unsupported builtin nonterminal parser: {}", name).as_slice()) diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index cbf34ba5eb320..381e4310d89ec 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -8,8 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use ast::{Ident, Matcher_, Matcher, MatchTok, MatchNonterminal, MatchSeq, TtDelimited}; -use ast::{TtSequence, TtToken}; +use ast::{Ident, TtDelimited, TtSequence, TtToken}; use ast; use codemap::{Span, DUMMY_SP}; use ext::base::{ExtCtxt, MacResult, MacroDef}; @@ -21,7 +20,7 @@ use parse::lexer::new_tt_reader; use parse::parser::Parser; use parse::attr::ParserAttr; use parse::token::{special_idents, gensym_ident}; -use parse::token::{MatchNt, NtMatchers, NtTT}; +use parse::token::{MatchNt, NtTT}; use parse::token; use print; use ptr::P; @@ -207,6 +206,11 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, cx.span_fatal(best_fail_spot, best_fail_msg.as_slice()); } +// Note that macro-by-example's input is also matched against a token tree: +// $( $lhs:tt => $rhs:tt );+ +// +// Holy self-referential! + /// This procedure performs the expansion of the /// macro_rules! macro. It parses the RHS and adds /// an extension to the current context. diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 75fbdb8d8b6b0..746e1d112c7e9 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -656,8 +656,6 @@ pub fn noop_fold_interpolated(nt: token::Nonterminal, fld: &mut T) token::NtMeta(meta_item) => token::NtMeta(fld.fold_meta_item(meta_item)), token::NtPath(box path) => token::NtPath(box fld.fold_path(path)), token::NtTT(tt) => token::NtTT(P(fld.fold_tt(&*tt))), - // it looks to me like we can leave out the matchers: token::NtMatchers(matchers) - _ => nt } } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 741014fec8962..5a0c7d92fa2cb 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -37,8 +37,8 @@ use ast::{ItemMac, ItemMod, ItemStruct, ItemTrait, ItemTy}; use ast::{LifetimeDef, Lit, Lit_}; use ast::{LitBool, LitChar, LitByte, LitBinary}; use ast::{LitNil, LitStr, LitInt, Local, LocalLet}; -use ast::{MutImmutable, MutMutable, Mac_, MacInvocTT, Matcher, MatchNonterminal, MatchNormal}; -use ast::{MatchSeq, MatchTok, Method, MutTy, BiMul, Mutability}; +use ast::{MutImmutable, MutMutable, Mac_, MacInvocTT, MatchNormal}; +use ast::{Method, MutTy, BiMul, Mutability}; use ast::{MethodImplItem, NamedField, UnNeg, NoReturn, UnNot}; use ast::{Pat, PatEnum, PatIdent, PatLit, PatRange, PatRegion, PatStruct}; use ast::{PatTup, PatBox, PatWild, PatWildMulti, PatWildSingle}; @@ -2628,66 +2628,6 @@ impl<'a> Parser<'a> { tts } - pub fn parse_matchers(&mut self) -> Vec { - // unification of Matcher's and TokenTree's would vastly improve - // the interpolation of Matcher's - maybe_whole!(self, NtMatchers); - let mut name_idx = 0u; - let delim = self.expect_open_delim(); - self.parse_matcher_subseq_upto(&mut name_idx, &token::CloseDelim(delim)) - } - - /// This goofy function is necessary to correctly match parens in Matcher's. - /// Otherwise, `$( ( )` would be a valid Matcher, and `$( () )` would be - /// invalid. It's similar to common::parse_seq. - pub fn parse_matcher_subseq_upto(&mut self, - name_idx: &mut uint, - ket: &token::Token) - -> Vec { - let mut ret_val = Vec::new(); - let mut lparens = 0u; - - while self.token != *ket || lparens > 0u { - if self.token == token::OpenDelim(token::Paren) { lparens += 1u; } - if self.token == token::CloseDelim(token::Paren) { lparens -= 1u; } - ret_val.push(self.parse_matcher(name_idx)); - } - - self.bump(); - - return ret_val; - } - - pub fn parse_matcher(&mut self, name_idx: &mut uint) -> Matcher { - let lo = self.span.lo; - - let m = if self.token == token::Dollar { - self.bump(); - if self.token == token::OpenDelim(token::Paren) { - let name_idx_lo = *name_idx; - self.bump(); - let ms = self.parse_matcher_subseq_upto(name_idx, - &token::CloseDelim(token::Paren)); - if ms.len() == 0u { - self.fatal("repetition body must be nonempty"); - } - let (sep, kleene_op) = self.parse_sep_and_kleene_op(); - MatchSeq(ms, sep, kleene_op, name_idx_lo, *name_idx) - } else { - let bound_to = self.parse_ident(); - self.expect(&token::Colon); - let nt_name = self.parse_ident(); - let m = MatchNonterminal(bound_to, nt_name, *name_idx); - *name_idx += 1; - m - } - } else { - MatchTok(self.bump_and_get()) - }; - - return spanned(lo, self.span.hi, m); - } - /// Parse a prefix-operator expr pub fn parse_prefix_expr(&mut self) -> P { let lo = self.span.lo; diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 8dd2f8b840f92..b0cca5e14de15 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -337,7 +337,6 @@ pub enum Nonterminal { NtMeta(P), NtPath(Box), NtTT(P), // needs P'ed to break a circularity - NtMatchers(Vec) } impl fmt::Show for Nonterminal { @@ -353,7 +352,6 @@ impl fmt::Show for Nonterminal { NtMeta(..) => f.pad("NtMeta(..)"), NtPath(..) => f.pad("NtPath(..)"), NtTT(..) => f.pad("NtTT(..)"), - NtMatchers(..) => f.pad("NtMatchers(..)"), } } } diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 12ce81eedca8c..8ffc2aa358380 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -272,7 +272,6 @@ pub fn token_to_string(tok: &Token) -> String { token::NtPat(..) => "an interpolated pattern".into_string(), token::NtIdent(..) => "an interpolated identifier".into_string(), token::NtTT(..) => "an interpolated tt".into_string(), - token::NtMatchers(..) => "an interpolated matcher sequence".into_string(), } } } From a8ce669b42471863862e84148286f86040f1f4d5 Mon Sep 17 00:00:00 2001 From: Piotr Czarnecki Date: Fri, 10 Oct 2014 19:47:20 +0100 Subject: [PATCH 3/5] Workaround to have doc comments desugared only in macros --- src/libsyntax/ext/tt/macro_rules.rs | 11 ++++++----- src/libsyntax/ext/tt/transcribe.rs | 14 ++++++++++++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 381e4310d89ec..15792b7f7716a 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -164,11 +164,12 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt, _ => cx.span_fatal(sp, "malformed macro lhs") }; // `None` is because we're not interpolating - let arg_rdr = new_tt_reader(&cx.parse_sess().span_diagnostic, - None, - arg.iter() - .map(|x| (*x).clone()) - .collect()); + let mut arg_rdr = new_tt_reader(&cx.parse_sess().span_diagnostic, + None, + arg.iter() + .map(|x| (*x).clone()) + .collect()); + arg_rdr.desugar_doc_comments = true; match parse(cx.parse_sess(), cx.cfg(), arg_rdr, lhs_tt) { Success(named_matches) => { let rhs = match *rhses[i] { diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index e8a2d9a243362..88253c0d24c61 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -43,6 +43,8 @@ pub struct TtReader<'a> { /* cached: */ pub cur_tok: Token, pub cur_span: Span, + /// Transform doc comments. Only useful in macro invocations + pub desugar_doc_comments: bool, } /// This can do Macro-By-Example transcription. On the other hand, if @@ -66,6 +68,7 @@ pub fn new_tt_reader<'a>(sp_diag: &'a SpanHandler, }, repeat_idx: Vec::new(), repeat_len: Vec::new(), + desugar_doc_comments: false, /* dummy values, never read: */ cur_tok: token::Eof, cur_span: DUMMY_SP, @@ -279,8 +282,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { } } // TtDelimited or any token that can be unzipped - seq @ TtDelimited(..) | seq @ TtToken(_, DocComment(..)) - | seq @ TtToken(_, MatchNt(..)) => { + seq @ TtDelimited(..) | seq @ TtToken(_, MatchNt(..)) => { // do not advance the idx yet r.stack.push(TtFrame { forest: seq.expand_into_tts(), @@ -290,6 +292,14 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { }); // if this could be 0-length, we'd need to potentially recur here } + TtToken(sp, DocComment(name)) if r.desugar_doc_comments => { + r.stack.push(TtFrame { + forest: TtToken(sp, DocComment(name)).expand_into_tts(), + idx: 0, + dotdotdoted: false, + sep: None + }); + } TtToken(sp, tok) => { r.cur_span = sp; r.cur_tok = tok; From 964191a313b84785b29b7a33560ae8959f66b582 Mon Sep 17 00:00:00 2001 From: Piotr Czarnecki Date: Sun, 12 Oct 2014 12:42:41 +0100 Subject: [PATCH 4/5] Add and update tests --- src/test/compile-fail/issue-6596.rs | 2 +- .../compile-fail/macro-match-nonterminal.rs | 17 ++++++++++++++ .../macro-of-higher-order.rs} | 22 +++++++------------ 3 files changed, 26 insertions(+), 15 deletions(-) create mode 100644 src/test/compile-fail/macro-match-nonterminal.rs rename src/test/{compile-fail/issue-5060-fail.rs => run-pass/macro-of-higher-order.rs} (51%) diff --git a/src/test/compile-fail/issue-6596.rs b/src/test/compile-fail/issue-6596.rs index 54bcf415d2ead..267b30677a18f 100644 --- a/src/test/compile-fail/issue-6596.rs +++ b/src/test/compile-fail/issue-6596.rs @@ -10,7 +10,7 @@ #![feature(macro_rules)] -// error-pattern: unknown macro variable `nonexistent` +// error-pattern: unexpected token macro_rules! e( ($inp:ident) => ( diff --git a/src/test/compile-fail/macro-match-nonterminal.rs b/src/test/compile-fail/macro-match-nonterminal.rs new file mode 100644 index 0000000000000..d6d32d94a2956 --- /dev/null +++ b/src/test/compile-fail/macro-match-nonterminal.rs @@ -0,0 +1,17 @@ +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![feature(macro_rules)] + +macro_rules! test ( ($a, $b) => (()); ) //~ ERROR Cannot transcribe + +fn main() { + test!() +} diff --git a/src/test/compile-fail/issue-5060-fail.rs b/src/test/run-pass/macro-of-higher-order.rs similarity index 51% rename from src/test/compile-fail/issue-5060-fail.rs rename to src/test/run-pass/macro-of-higher-order.rs index fedb064d2e5f5..561933d759927 100644 --- a/src/test/compile-fail/issue-5060-fail.rs +++ b/src/test/run-pass/macro-of-higher-order.rs @@ -1,4 +1,4 @@ -// Copyright 2013 The Rust Project Developers. See the COPYRIGHT +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT // file at the top-level directory of this distribution and at // http://rust-lang.org/COPYRIGHT. // @@ -10,20 +10,14 @@ #![feature(macro_rules)] -macro_rules! print_hd_tl ( - ($field_hd:ident, $($field_tl:ident),+) => ({ - print!("{}", stringify!($field)); //~ ERROR unknown macro variable - print!("::["); - $( - print!("{}", stringify!($field_tl)); - print!(", "); - )+ - // FIXME: #9970 - print!("{}", "]\n"); - }) +macro_rules! higher_order ( + (subst $lhs:tt => $rhs:tt) => ({ + macro_rules! anon ( $lhs => $rhs ) + anon!(1u, 2u, "foo") + }); ) fn main() { - print_hd_tl!(x, y, z, w) + let val = higher_order!(subst ($x:expr, $y:expr, $foo:expr) => (($x + $y, $foo))); + assert_eq!(val, (3, "foo")); } - From 00676c8ea20a7310dacc85759daf57eab86ac965 Mon Sep 17 00:00:00 2001 From: Piotr Czarnecki Date: Sun, 2 Nov 2014 12:21:16 +0100 Subject: [PATCH 5/5] Add `ast::SequenceRepetition` --- src/libsyntax/ast.rs | 95 ++++++++++++++++++---------- src/libsyntax/ext/tt/macro_parser.rs | 72 +++++++++++++-------- src/libsyntax/ext/tt/macro_rules.rs | 26 ++++---- src/libsyntax/ext/tt/transcribe.rs | 30 +++++---- src/libsyntax/fold.rs | 11 ++-- src/libsyntax/parse/parser.rs | 11 +++- src/libsyntax/print/pprust.rs | 8 +-- 7 files changed, 160 insertions(+), 93 deletions(-) diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index 639b007e7ab24..3ebbede82ab4a 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -627,6 +627,19 @@ impl Delimited { } } +/// A sequence of token treesee +#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)] +pub struct SequenceRepetition { + /// The sequence of token trees + pub tts: Vec, + /// The optional separator + pub separator: Option, + /// Whether the sequence can be repeated zero (*), or one or more times (+) + pub op: KleeneOp, + /// The number of `MatchNt`s that appear in the sequence (and subsequences) + pub num_captures: uint, +} + /// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star) /// for token sequences. #[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)] @@ -657,58 +670,76 @@ pub enum TokenTree { // This only makes sense in MBE macros. - /// A kleene-style repetition sequence with a span, a TT forest, - /// an optional separator, and a boolean where true indicates - /// zero or more (..), and false indicates one or more (+). - /// The last member denotes the number of `MATCH_NONTERMINAL`s - /// in the forest. - // FIXME(eddyb) #12938 Use Rc<[TokenTree]> after DST. - TtSequence(Span, Rc>, Option<::parse::token::Token>, KleeneOp, uint), + /// A kleene-style repetition sequence with a span + // FIXME(eddyb) #12938 Use DST. + TtSequence(Span, Rc), } impl TokenTree { - /// For unrolling some tokens or token trees into equivalent sequences. - pub fn expand_into_tts(self) -> Rc> { - match self { - TtToken(sp, token::DocComment(name)) => { + pub fn len(&self) -> uint { + match *self { + TtToken(_, token::DocComment(_)) => 2, + TtToken(_, token::SubstNt(..)) => 2, + TtToken(_, token::MatchNt(..)) => 3, + TtDelimited(_, ref delimed) => { + delimed.tts.len() + 2 + } + TtSequence(_, ref seq) => { + seq.tts.len() + } + TtToken(..) => 0 + } + } + + pub fn get_tt(&self, index: uint) -> TokenTree { + match (self, index) { + (&TtToken(sp, token::DocComment(_)), 0) => { + TtToken(sp, token::Pound) + } + (&TtToken(sp, token::DocComment(name)), 1) => { let doc = MetaNameValue(token::intern_and_get_ident("doc"), respan(sp, LitStr(token::get_name(name), CookedStr))); let doc = token::NtMeta(P(respan(sp, doc))); - let delimed = Delimited { + TtDelimited(sp, Rc::new(Delimited { delim: token::Bracket, open_span: sp, tts: vec![TtToken(sp, token::Interpolated(doc))], close_span: sp, - }; - Rc::new(vec![TtToken(sp, token::Pound), - TtDelimited(sp, Rc::new(delimed))]) + })) } - TtDelimited(_, ref delimed) => { - let mut tts = Vec::with_capacity(1 + delimed.tts.len() + 1); - tts.push(delimed.open_tt()); - tts.extend(delimed.tts.iter().map(|tt| tt.clone())); - tts.push(delimed.close_tt()); - Rc::new(tts) + (&TtDelimited(_, ref delimed), _) => { + if index == 0 { + return delimed.open_tt(); + } + if index == delimed.tts.len() + 1 { + return delimed.close_tt(); + } + delimed.tts[index - 1].clone() + } + (&TtToken(sp, token::SubstNt(name, name_st)), _) => { + let v = [TtToken(sp, token::Dollar), + TtToken(sp, token::Ident(name, name_st))]; + v[index] } - TtToken(sp, token::SubstNt(name, namep)) => { - Rc::new(vec![TtToken(sp, token::Dollar), - TtToken(sp, token::Ident(name, namep))]) + (&TtToken(sp, token::MatchNt(name, kind, name_st, kind_st)), _) => { + let v = [TtToken(sp, token::SubstNt(name, name_st)), + TtToken(sp, token::Colon), + TtToken(sp, token::Ident(kind, kind_st))]; + v[index] } - TtToken(sp, token::MatchNt(name, kind, namep, kindp)) => { - Rc::new(vec![TtToken(sp, token::SubstNt(name, namep)), - TtToken(sp, token::Colon), - TtToken(sp, token::Ident(kind, kindp))]) + (&TtSequence(_, ref seq), _) => { + seq.tts[index].clone() } - _ => panic!("Cannot expand a token") + _ => panic!("Cannot expand a token tree") } } /// Returns the `Span` corresponding to this token tree. pub fn get_span(&self) -> Span { match *self { - TtToken(span, _) => span, - TtDelimited(span, _) => span, - TtSequence(span, _, _, _, _) => span, + TtToken(span, _) => span, + TtDelimited(span, _) => span, + TtSequence(span, _) => span, } } } diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index 833211f53e7aa..1f0b667259473 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -100,17 +100,39 @@ use std::collections::hash_map::{Vacant, Occupied}; // To avoid costly uniqueness checks, we require that `MatchSeq` always has // a nonempty body. +#[deriving(Clone)] +enum TokenTreeOrTokenTreeVec { + Tt(ast::TokenTree), + TtSeq(Rc>), +} + +impl TokenTreeOrTokenTreeVec { + fn len(&self) -> uint { + match self { + &TtSeq(ref v) => v.len(), + &Tt(ref tt) => tt.len(), + } + } + + fn get_tt(&self, index: uint) -> TokenTree { + match self { + &TtSeq(ref v) => v[index].clone(), + &Tt(ref tt) => tt.get_tt(index), + } + } +} + /// an unzipping of `TokenTree`s #[deriving(Clone)] struct MatcherTtFrame { - elts: Rc>, + elts: TokenTreeOrTokenTreeVec, idx: uint, } #[deriving(Clone)] pub struct MatcherPos { stack: Vec, - elts: Rc>, + top_elts: TokenTreeOrTokenTreeVec, sep: Option, idx: uint, up: Option>, @@ -124,8 +146,8 @@ pub struct MatcherPos { pub fn count_names(ms: &[TokenTree]) -> uint { ms.iter().fold(0, |count, elt| { count + match elt { - &TtSequence(_, _, _, _, advance_by) => { - advance_by + &TtSequence(_, ref seq) => { + seq.num_captures } &TtDelimited(_, ref delim) => { count_names(delim.tts.as_slice()) @@ -144,7 +166,7 @@ pub fn initial_matcher_pos(ms: Rc>, sep: Option, lo: ByteP let matches = Vec::from_fn(match_idx_hi, |_i| Vec::new()); box MatcherPos { stack: vec![], - elts: ms, + top_elts: TtSeq(ms), sep: sep, idx: 0u, up: None, @@ -183,8 +205,8 @@ pub fn nameize(p_s: &ParseSess, ms: &[TokenTree], res: &[Rc]) fn n_rec(p_s: &ParseSess, m: &TokenTree, res: &[Rc], ret_val: &mut HashMap>, idx: &mut uint) { match m { - &TtSequence(_, ref more_ms, _, _, _) => { - for next_m in more_ms.iter() { + &TtSequence(_, ref seq) => { + for next_m in seq.tts.iter() { n_rec(p_s, next_m, res, ret_val, idx) } } @@ -278,10 +300,10 @@ pub fn parse(sess: &ParseSess, }; // When unzipped trees end, remove them - while ei.idx >= ei.elts.len() { + while ei.idx >= ei.top_elts.len() { match ei.stack.pop() { Some(MatcherTtFrame { elts, idx }) => { - ei.elts = elts; + ei.top_elts = elts; ei.idx = idx + 1; } None => break @@ -289,7 +311,7 @@ pub fn parse(sess: &ParseSess, } let idx = ei.idx; - let len = ei.elts.len(); + let len = ei.top_elts.len(); /* at end of sequence */ if idx >= len { @@ -352,17 +374,16 @@ pub fn parse(sess: &ParseSess, eof_eis.push(ei); } } else { - match (*ei.elts)[idx].clone() { + match ei.top_elts.get_tt(idx) { /* need to descend into sequence */ - TtSequence(_, ref matchers, ref sep, kleene_op, match_num) => { - if kleene_op == ast::ZeroOrMore { + TtSequence(sp, seq) => { + if seq.op == ast::ZeroOrMore { let mut new_ei = ei.clone(); - new_ei.match_cur += match_num; + new_ei.match_cur += seq.num_captures; new_ei.idx += 1u; //we specifically matched zero repeats. - for idx in range(ei.match_cur, ei.match_cur + match_num) { - new_ei.matches[idx] - .push(Rc::new(MatchedSeq(Vec::new(), sp))); + for idx in range(ei.match_cur, ei.match_cur + seq.num_captures) { + new_ei.matches[idx].push(Rc::new(MatchedSeq(Vec::new(), sp))); } cur_eis.push(new_ei); @@ -372,15 +393,15 @@ pub fn parse(sess: &ParseSess, let ei_t = ei; cur_eis.push(box MatcherPos { stack: vec![], - elts: matchers.clone(), - sep: (*sep).clone(), + sep: seq.separator.clone(), idx: 0u, matches: matches, match_lo: ei_t.match_cur, match_cur: ei_t.match_cur, - match_hi: ei_t.match_cur + match_num, + match_hi: ei_t.match_cur + seq.num_captures, up: Some(ei_t), - sp_lo: sp.lo + sp_lo: sp.lo, + top_elts: Tt(TtSequence(sp, seq)), }); } TtToken(_, MatchNt(..)) => { @@ -395,11 +416,10 @@ pub fn parse(sess: &ParseSess, return Error(sp, "Cannot transcribe in macro LHS".into_string()) } seq @ TtDelimited(..) | seq @ TtToken(_, DocComment(..)) => { - let tts = seq.expand_into_tts(); - let elts = mem::replace(&mut ei.elts, tts); + let lower_elts = mem::replace(&mut ei.top_elts, Tt(seq)); let idx = ei.idx; ei.stack.push(MatcherTtFrame { - elts: elts, + elts: lower_elts, idx: idx, }); ei.idx = 0; @@ -433,7 +453,7 @@ pub fn parse(sess: &ParseSess, if (bb_eis.len() > 0u && next_eis.len() > 0u) || bb_eis.len() > 1u { let nts = bb_eis.iter().map(|ei| { - match (*ei.elts)[ei.idx] { + match ei.top_elts.get_tt(ei.idx) { TtToken(_, MatchNt(bind, name, _, _)) => { (format!("{} ('{}')", token::get_ident(name), @@ -458,7 +478,7 @@ pub fn parse(sess: &ParseSess, let mut rust_parser = Parser::new(sess, cfg.clone(), box rdr.clone()); let mut ei = bb_eis.pop().unwrap(); - match (*ei.elts)[ei.idx] { + match ei.top_elts.get_tt(ei.idx) { TtToken(_, MatchNt(_, name, _, _)) => { let name_string = token::get_ident(name); let match_cur = ei.match_cur; diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index 15792b7f7716a..92c68b7a9c724 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -233,20 +233,24 @@ pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt, let match_rhs_tok = MatchNt(rhs_nm, special_idents::tt, token::Plain, token::Plain); let argument_gram = vec!( TtSequence(DUMMY_SP, - Rc::new(vec![ - TtToken(DUMMY_SP, match_lhs), - TtToken(DUMMY_SP, token::FatArrow), - TtToken(DUMMY_SP, match_rhs)]), - Some(token::Semi), - ast::OneOrMore, - 2), + Rc::new(ast::SequenceRepetition { + tts: vec![ + TtToken(DUMMY_SP, match_lhs_tok), + TtToken(DUMMY_SP, token::FatArrow), + TtToken(DUMMY_SP, match_rhs_tok)], + separator: Some(token::Semi), + op: ast::OneOrMore, + num_captures: 2 + })), //to phase into semicolon-termination instead of //semicolon-separation TtSequence(DUMMY_SP, - Rc::new(vec![TtToken(DUMMY_SP, token::Semi)]), - None, - ast::ZeroOrMore, - 0)); + Rc::new(ast::SequenceRepetition { + tts: vec![TtToken(DUMMY_SP, token::Semi)], + separator: None, + op: ast::ZeroOrMore, + num_captures: 0 + }))); // Parse the macro_rules! invocation (`none` is for no interpolations): diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 88253c0d24c61..5842afe11ce2c 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -25,7 +25,7 @@ use std::collections::HashMap; ///an unzipping of `TokenTree`s #[deriving(Clone)] struct TtFrame { - forest: Rc>, + forest: TokenTree, idx: uint, dotdotdoted: bool, sep: Option, @@ -57,7 +57,11 @@ pub fn new_tt_reader<'a>(sp_diag: &'a SpanHandler, let mut r = TtReader { sp_diag: sp_diag, stack: vec!(TtFrame { - forest: Rc::new(src), + forest: TtSequence(DUMMY_SP, Rc::new(ast::SequenceRepetition { + tts: src, + // doesn't matter. This merely holds the root unzipping. + separator: None, op: ast::ZeroOrMore, num_captures: 0 + })), idx: 0, dotdotdoted: false, sep: None, @@ -129,8 +133,8 @@ fn lockstep_iter_size(t: &TokenTree, r: &TtReader) -> LockstepIterSize { size + lockstep_iter_size(tt, r) }) }, - TtSequence(_, ref tts, _, _, _) => { - tts.iter().fold(LisUnconstrained, |size, tt| { + TtSequence(_, ref seq) => { + seq.tts.iter().fold(LisUnconstrained, |size, tt| { size + lockstep_iter_size(tt, r) }) }, @@ -202,12 +206,12 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { let t = { let frame = r.stack.last().unwrap(); // FIXME(pcwalton): Bad copy. - (*frame.forest)[frame.idx].clone() + frame.forest.get_tt(frame.idx) }; match t { - TtSequence(sp, tts, sep, kleene_op, n) => { + TtSequence(sp, seq) => { // FIXME(pcwalton): Bad copy. - match lockstep_iter_size(&TtSequence(sp, tts.clone(), sep.clone(), kleene_op, n), + match lockstep_iter_size(&TtSequence(sp, seq.clone()), r) { LisUnconstrained => { r.sp_diag.span_fatal( @@ -222,7 +226,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { } LisConstraint(len, _) => { if len == 0 { - if kleene_op == ast::OneOrMore { + if seq.op == ast::OneOrMore { // FIXME #2887 blame invoker r.sp_diag.span_fatal(sp.clone(), "this must repeat at least once"); @@ -234,10 +238,10 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { r.repeat_len.push(len); r.repeat_idx.push(0); r.stack.push(TtFrame { - forest: tts, idx: 0, dotdotdoted: true, - sep: sep.clone() + sep: seq.separator.clone(), + forest: TtSequence(sp, seq), }); } } @@ -247,7 +251,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { match lookup_cur_matched(r, ident) { None => { r.stack.push(TtFrame { - forest: TtToken(sp, SubstNt(ident, namep)).expand_into_tts(), + forest: TtToken(sp, SubstNt(ident, namep)), idx: 0, dotdotdoted: false, sep: None @@ -285,7 +289,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { seq @ TtDelimited(..) | seq @ TtToken(_, MatchNt(..)) => { // do not advance the idx yet r.stack.push(TtFrame { - forest: seq.expand_into_tts(), + forest: seq, idx: 0, dotdotdoted: false, sep: None @@ -294,7 +298,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan { } TtToken(sp, DocComment(name)) if r.desugar_doc_comments => { r.stack.push(TtFrame { - forest: TtToken(sp, DocComment(name)).expand_into_tts(), + forest: TtToken(sp, DocComment(name)), idx: 0, dotdotdoted: false, sep: None diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 746e1d112c7e9..1d4ac0edc3639 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -581,12 +581,13 @@ pub fn noop_fold_tt(tt: &TokenTree, fld: &mut T) -> TokenTree { } )) }, - TtSequence(span, ref pattern, ref sep, is_optional, advance_by) => + TtSequence(span, ref seq) => TtSequence(span, - Rc::new(fld.fold_tts(pattern.as_slice())), - sep.clone().map(|tok| fld.fold_token(tok)), - is_optional, - advance_by), + Rc::new(SequenceRepetition { + tts: fld.fold_tts(seq.tts.as_slice()), + separator: seq.separator.clone().map(|tok| fld.fold_token(tok)), + ..**seq + })), } } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 5a0c7d92fa2cb..08a1001a30beb 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -48,7 +48,8 @@ use ast::{StmtExpr, StmtSemi, StmtMac, StructDef, StructField}; use ast::{StructVariantKind, BiSub}; use ast::StrStyle; use ast::{SelfExplicit, SelfRegion, SelfStatic, SelfValue}; -use ast::{Delimited, TokenTree, TraitItem, TraitRef, TtDelimited, TtSequence, TtToken}; +use ast::{Delimited, SequenceRepetition, TokenTree, TraitItem, TraitRef}; +use ast::{TtDelimited, TtSequence, TtToken}; use ast::{TupleVariantKind, Ty, Ty_, TyBot}; use ast::{TypeField, TyFixedLengthVec, TyClosure, TyProc, TyBareFn}; use ast::{TyTypeof, TyInfer, TypeMethod}; @@ -2551,7 +2552,13 @@ impl<'a> Parser<'a> { Spanned { node, .. } => node, }; let name_num = macro_parser::count_names(seq.as_slice()); - TtSequence(mk_sp(sp.lo, p.span.hi), Rc::new(seq), sep, repeat, name_num) + TtSequence(mk_sp(sp.lo, p.span.hi), + Rc::new(SequenceRepetition { + tts: seq, + separator: sep, + op: repeat, + num_captures: name_num + })) } else { // A nonterminal that matches or not let namep = match p.token { token::Ident(_, p) => p, _ => token::Plain }; diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 8ffc2aa358380..bd4b70bc52cf3 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -1137,19 +1137,19 @@ impl<'a> State<'a> { try!(space(&mut self.s)); word(&mut self.s, token_to_string(&delimed.close_token()).as_slice()) }, - ast::TtSequence(_, ref tts, ref separator, kleene_op, _) => { + ast::TtSequence(_, ref seq) => { try!(word(&mut self.s, "$(")); - for tt_elt in (*tts).iter() { + for tt_elt in seq.tts.iter() { try!(self.print_tt(tt_elt)); } try!(word(&mut self.s, ")")); - match *separator { + match seq.separator { Some(ref tk) => { try!(word(&mut self.s, token_to_string(tk).as_slice())); } None => {}, } - match kleene_op { + match seq.op { ast::ZeroOrMore => word(&mut self.s, "*"), ast::OneOrMore => word(&mut self.s, "+"), }