Skip to content

Improve interpolation of macro lhs and doc comments. Remove Matchers #17830

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 7, 2014
Merged

Improve interpolation of macro lhs and doc comments. Remove Matchers #17830

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
38ce6d9
Use `TokenTree`s in lhs of macros
pczarn Oct 6, 2014
6f30a4e
Remove `Matcher`s
pczarn Oct 6, 2014
a8ce669
Workaround to have doc comments desugared only in macros
pczarn Oct 10, 2014
964191a
Add and update tests
pczarn Oct 12, 2014
00676c8
Add `ast::SequenceRepetition`
pczarn Nov 2, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/librustdoc/html/highlight.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,8 @@ fn doit(sess: &parse::ParseSess, mut lexer: lexer::StringReader,

token::Lifetime(..) => "lifetime",
token::DocComment(..) => "doccomment",
token::Underscore | token::Eof | token::Interpolated(..) => "",
token::Underscore | token::Eof | token::Interpolated(..) |
token::MatchNt(..) | token::SubstNt(..) => "",
};

// as mentioned above, use the original source code instead of
Expand Down
163 changes: 83 additions & 80 deletions src/libsyntax/ast.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

// The Rust abstract syntax tree.

use codemap::{Span, Spanned, DUMMY_SP, ExpnId};
use codemap::{Span, Spanned, DUMMY_SP, ExpnId, respan};
use abi::Abi;
use ast_util;
use owned_slice::OwnedSlice;
Expand Down Expand Up @@ -627,6 +627,19 @@ impl Delimited {
}
}

/// A sequence of token treesee
#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
pub struct SequenceRepetition {
/// The sequence of token trees
pub tts: Vec<TokenTree>,
/// The optional separator
pub separator: Option<token::Token>,
/// Whether the sequence can be repeated zero (*), or one or more times (+)
pub op: KleeneOp,
/// The number of `MatchNt`s that appear in the sequence (and subsequences)
pub num_captures: uint,
}

/// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star)
/// for token sequences.
#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
Expand All @@ -641,14 +654,12 @@ pub enum KleeneOp {
/// be passed to syntax extensions using a uniform type.
///
/// If the syntax extension is an MBE macro, it will attempt to match its
/// LHS "matchers" against the provided token tree, and if it finds a
/// LHS token tree against the provided token tree, and if it finds a
/// match, will transcribe the RHS token tree, splicing in any captured
/// `macro_parser::matched_nonterminals` into the `TtNonterminal`s it finds.
/// macro_parser::matched_nonterminals into the `SubstNt`s it finds.
///
/// The RHS of an MBE macro is the only place a `TtNonterminal` or `TtSequence`
/// makes any real sense. You could write them elsewhere but nothing
/// else knows what to do with them, so you'll probably get a syntax
/// error.
/// The RHS of an MBE macro is the only place `SubstNt`s are substituted.
/// Nothing special happens to misnamed or misplaced `SubstNt`s.
#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
#[doc="For macro invocations; parsing is delegated to the macro"]
pub enum TokenTree {
Expand All @@ -657,90 +668,82 @@ pub enum TokenTree {
/// A delimited sequence of token trees
TtDelimited(Span, Rc<Delimited>),

// These only make sense for right-hand-sides of MBE macros:
// This only makes sense in MBE macros.

/// A Kleene-style repetition sequence with an optional separator.
// FIXME(eddyb) #6308 Use Rc<[TokenTree]> after DST.
TtSequence(Span, Rc<Vec<TokenTree>>, Option<token::Token>, KleeneOp),
/// A syntactic variable that will be filled in by macro expansion.
TtNonterminal(Span, Ident)
/// A kleene-style repetition sequence with a span
// FIXME(eddyb) #12938 Use DST.
TtSequence(Span, Rc<SequenceRepetition>),
}

impl TokenTree {
pub fn len(&self) -> uint {
match *self {
TtToken(_, token::DocComment(_)) => 2,
TtToken(_, token::SubstNt(..)) => 2,
TtToken(_, token::MatchNt(..)) => 3,
TtDelimited(_, ref delimed) => {
delimed.tts.len() + 2
}
TtSequence(_, ref seq) => {
seq.tts.len()
}
TtToken(..) => 0
}
}

pub fn get_tt(&self, index: uint) -> TokenTree {
match (self, index) {
(&TtToken(sp, token::DocComment(_)), 0) => {
TtToken(sp, token::Pound)
}
(&TtToken(sp, token::DocComment(name)), 1) => {
let doc = MetaNameValue(token::intern_and_get_ident("doc"),
respan(sp, LitStr(token::get_name(name), CookedStr)));
let doc = token::NtMeta(P(respan(sp, doc)));
TtDelimited(sp, Rc::new(Delimited {
delim: token::Bracket,
open_span: sp,
tts: vec![TtToken(sp, token::Interpolated(doc))],
close_span: sp,
}))
}
(&TtDelimited(_, ref delimed), _) => {
if index == 0 {
return delimed.open_tt();
}
if index == delimed.tts.len() + 1 {
return delimed.close_tt();
}
delimed.tts[index - 1].clone()
}
(&TtToken(sp, token::SubstNt(name, name_st)), _) => {
let v = [TtToken(sp, token::Dollar),
TtToken(sp, token::Ident(name, name_st))];
v[index]
}
(&TtToken(sp, token::MatchNt(name, kind, name_st, kind_st)), _) => {
let v = [TtToken(sp, token::SubstNt(name, name_st)),
TtToken(sp, token::Colon),
TtToken(sp, token::Ident(kind, kind_st))];
v[index]
}
(&TtSequence(_, ref seq), _) => {
seq.tts[index].clone()
}
_ => panic!("Cannot expand a token tree")
}
}

/// Returns the `Span` corresponding to this token tree.
pub fn get_span(&self) -> Span {
match *self {
TtToken(span, _) => span,
TtDelimited(span, _) => span,
TtSequence(span, _, _, _) => span,
TtNonterminal(span, _) => span,
TtToken(span, _) => span,
TtDelimited(span, _) => span,
TtSequence(span, _) => span,
}
}
}

// Matchers are nodes defined-by and recognized-by the main rust parser and
// language, but they're only ever found inside syntax-extension invocations;
// indeed, the only thing that ever _activates_ the rules in the rust parser
// for parsing a matcher is a matcher looking for the 'matchers' nonterminal
// itself. Matchers represent a small sub-language for pattern-matching
// token-trees, and are thus primarily used by the macro-defining extension
// itself.
//
// MatchTok
// --------
//
// A matcher that matches a single token, denoted by the token itself. So
// long as there's no $ involved.
//
//
// MatchSeq
// --------
//
// A matcher that matches a sequence of sub-matchers, denoted various
// possible ways:
//
// $(M)* zero or more Ms
// $(M)+ one or more Ms
// $(M),+ one or more comma-separated Ms
// $(A B C);* zero or more semi-separated 'A B C' seqs
//
//
// MatchNonterminal
// -----------------
//
// A matcher that matches one of a few interesting named rust
// nonterminals, such as types, expressions, items, or raw token-trees. A
// black-box matcher on expr, for example, binds an expr to a given ident,
// and that ident can re-occur as an interpolation in the RHS of a
// macro-by-example rule. For example:
//
// $foo:expr => 1 + $foo // interpolate an expr
// $foo:tt => $foo // interpolate a token-tree
// $foo:tt => bar! $foo // only other valid interpolation
// // is in arg position for another
// // macro
//
// As a final, horrifying aside, note that macro-by-example's input is
// also matched by one of these matchers. Holy self-referential! It is matched
// by a MatchSeq, specifically this one:
//
// $( $lhs:matchers => $rhs:tt );+
//
// If you understand that, you have closed the loop and understand the whole
// macro system. Congratulations.
pub type Matcher = Spanned<Matcher_>;

#[deriving(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Show)]
pub enum Matcher_ {
/// Match one token
MatchTok(token::Token),
/// Match repetitions of a sequence: body, separator, Kleene operator,
/// lo, hi position-in-match-array used:
MatchSeq(Vec<Matcher>, Option<token::Token>, KleeneOp, uint, uint),
/// Parse a Rust NT: name to bind, name of NT, position in match array:
MatchNonterminal(Ident, Ident, uint)
}

pub type Mac = Spanned<Mac_>;

/// Represents a macro invocation. The Path indicates which macro
Expand Down
52 changes: 33 additions & 19 deletions src/libsyntax/ext/quote.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -616,6 +616,20 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
vec!(mk_name(cx, sp, ident.ident())));
}

token::MatchNt(name, kind, name_style, kind_style) => {
return cx.expr_call(sp,
mk_token_path(cx, sp, "MatchNt"),
vec![mk_ident(cx, sp, name),
mk_ident(cx, sp, kind),
match name_style {
ModName => mk_token_path(cx, sp, "ModName"),
Plain => mk_token_path(cx, sp, "Plain"),
},
match kind_style {
ModName => mk_token_path(cx, sp, "ModName"),
Plain => mk_token_path(cx, sp, "Plain"),
}]);
}
token::Interpolated(_) => panic!("quote! with interpolated token"),

_ => ()
Expand Down Expand Up @@ -654,6 +668,25 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {

fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
match *tt {
ast::TtToken(sp, SubstNt(ident, _)) => {
// tt.extend($ident.to_tokens(ext_cx).into_iter())

let e_to_toks =
cx.expr_method_call(sp,
cx.expr_ident(sp, ident),
id_ext("to_tokens"),
vec!(cx.expr_ident(sp, id_ext("ext_cx"))));
let e_to_toks =
cx.expr_method_call(sp, e_to_toks, id_ext("into_iter"), vec![]);

let e_push =
cx.expr_method_call(sp,
cx.expr_ident(sp, id_ext("tt")),
id_ext("extend"),
vec!(e_to_toks));

vec!(cx.stmt_expr(e_push))
}
ast::TtToken(sp, ref tok) => {
let e_sp = cx.expr_ident(sp, id_ext("_sp"));
let e_tok = cx.expr_call(sp,
Expand All @@ -673,25 +706,6 @@ fn mk_tt(cx: &ExtCtxt, _: Span, tt: &ast::TokenTree) -> Vec<P<ast::Stmt>> {
.collect()
},
ast::TtSequence(..) => panic!("TtSequence in quote!"),
ast::TtNonterminal(sp, ident) => {
// tt.extend($ident.to_tokens(ext_cx).into_iter())

let e_to_toks =
cx.expr_method_call(sp,
cx.expr_ident(sp, ident),
id_ext("to_tokens"),
vec!(cx.expr_ident(sp, id_ext("ext_cx"))));
let e_to_toks =
cx.expr_method_call(sp, e_to_toks, id_ext("into_iter"), vec![]);

let e_push =
cx.expr_method_call(sp,
cx.expr_ident(sp, id_ext("tt")),
id_ext("extend"),
vec!(e_to_toks));

vec!(cx.stmt_expr(e_push))
},
}
}

Expand Down
Loading