Skip to content

Commit 115f10b

Browse files
nnethercotennethercote
committed
Remove NtIdent and NtLifetime.
There's no need to use Interpolated for these because they both consist of a single token, and so there's no operator precedence problems with inserting them directly into the token stream.
1 parent baaa3b6 commit 115f10b

File tree

21 files changed

+109
-144
lines changed

21 files changed

+109
-144
lines changed

‎compiler/rustc_ast/src/ast_like.rs

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,7 @@ impl AstLike for crate::token::Nonterminal {
5151
| Nonterminal::NtMeta(_)
5252
| Nonterminal::NtPath(_)
5353
| Nonterminal::NtVis(_)
54-
| Nonterminal::NtBlock(_)
55-
| Nonterminal::NtIdent(..)
56-
| Nonterminal::NtLifetime(_) => &[],
54+
| Nonterminal::NtBlock(_) => &[],
5755
}
5856
}
5957
fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec<Attribute>)) {
@@ -66,9 +64,7 @@ impl AstLike for crate::token::Nonterminal {
6664
| Nonterminal::NtMeta(_)
6765
| Nonterminal::NtPath(_)
6866
| Nonterminal::NtVis(_)
69-
| Nonterminal::NtBlock(_)
70-
| Nonterminal::NtIdent(..)
71-
| Nonterminal::NtLifetime(_) => {}
67+
| Nonterminal::NtBlock(_) => {}
7268
}
7369
}
7470
fn tokens_mut(&mut self) -> Option<&mut Option<LazyTokenStream>> {
@@ -82,7 +78,6 @@ impl AstLike for crate::token::Nonterminal {
8278
Nonterminal::NtPath(path) => path.tokens_mut(),
8379
Nonterminal::NtVis(vis) => vis.tokens_mut(),
8480
Nonterminal::NtBlock(block) => block.tokens_mut(),
85-
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
8681
}
8782
}
8883
}

‎compiler/rustc_ast/src/attr/mod.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -408,7 +408,7 @@ impl MetaItem {
408408
I: Iterator<Item = TokenTree>,
409409
{
410410
// FIXME: Share code with `parse_path`.
411-
let path = match tokens.next().map(TokenTree::uninterpolate) {
411+
let path = match tokens.next() {
412412
Some(TokenTree::Token(Token {
413413
kind: kind @ (token::Ident(..) | token::ModSep),
414414
span,
@@ -426,7 +426,7 @@ impl MetaItem {
426426
};
427427
loop {
428428
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) =
429-
tokens.next().map(TokenTree::uninterpolate)
429+
tokens.next()
430430
{
431431
segments.push(PathSegment::from_ident(Ident::new(name, span)));
432432
} else {
@@ -449,6 +449,7 @@ impl MetaItem {
449449
},
450450
_ => return None,
451451
};
452+
//eprintln!("A1 {:?}", path);
452453
let list_closing_paren_pos = tokens.peek().map(|tt| tt.span().hi());
453454
let kind = MetaItemKind::from_tokens(tokens)?;
454455
let hi = match kind {

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -787,8 +787,6 @@ pub fn visit_interpolated<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut
787787
token::NtPat(pat) => vis.visit_pat(pat),
788788
token::NtExpr(expr) => vis.visit_expr(expr),
789789
token::NtTy(ty) => vis.visit_ty(ty),
790-
token::NtIdent(ident, _is_raw) => vis.visit_ident(ident),
791-
token::NtLifetime(ident) => vis.visit_ident(ident),
792790
token::NtLiteral(expr) => vis.visit_expr(expr),
793791
token::NtMeta(item) => {
794792
let AttrItem { path, args, tokens } = item.deref_mut();

‎compiler/rustc_ast/src/token.rs

Lines changed: 9 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ use rustc_macros::HashStable_Generic;
1212
use rustc_span::symbol::{kw, sym};
1313
use rustc_span::symbol::{Ident, Symbol};
1414
use rustc_span::{self, edition::Edition, Span, DUMMY_SP};
15-
use std::borrow::Cow;
1615
use std::{fmt, mem};
1716

1817
#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
@@ -227,14 +226,8 @@ pub enum TokenKind {
227226
Literal(Lit),
228227

229228
/// Identifier token.
230-
/// Do not forget about `NtIdent` when you want to match on identifiers.
231-
/// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
232-
/// treat regular and interpolated identifiers in the same way.
233229
Ident(Symbol, /* is_raw */ bool),
234230
/// Lifetime identifier token.
235-
/// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
236-
/// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
237-
/// treat regular and interpolated lifetime identifiers in the same way.
238231
Lifetime(Symbol),
239232

240233
Interpolated(Lrc<Nonterminal>),
@@ -364,7 +357,7 @@ impl Token {
364357

365358
/// Returns `true` if the token can appear at the start of an expression.
366359
pub fn can_begin_expr(&self) -> bool {
367-
match self.uninterpolate().kind {
360+
match self.kind {
368361
Ident(name, is_raw) =>
369362
ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
370363
OpenDelim(..) | // tuple, array or block
@@ -391,7 +384,7 @@ impl Token {
391384

392385
/// Returns `true` if the token can appear at the start of a type.
393386
pub fn can_begin_type(&self) -> bool {
394-
match self.uninterpolate().kind {
387+
match self.kind {
395388
Ident(name, is_raw) =>
396389
ident_can_begin_type(name, self.span, is_raw), // type name or keyword
397390
OpenDelim(Delimiter::Parenthesis) | // tuple
@@ -439,7 +432,7 @@ impl Token {
439432
///
440433
/// Keep this in sync with and `Lit::from_token`, excluding unary negation.
441434
pub fn can_begin_literal_maybe_minus(&self) -> bool {
442-
match self.uninterpolate().kind {
435+
match self.kind {
443436
Literal(..) | BinOp(Minus) => true,
444437
Ident(name, false) if name.is_bool_lit() => true,
445438
Interpolated(ref nt) => match &**nt {
@@ -457,37 +450,18 @@ impl Token {
457450
}
458451
}
459452

460-
// A convenience function for matching on identifiers during parsing.
461-
// Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token
462-
// into the regular identifier or lifetime token it refers to,
463-
// otherwise returns the original token.
464-
pub fn uninterpolate(&self) -> Cow<'_, Token> {
465-
match &self.kind {
466-
Interpolated(nt) => match **nt {
467-
NtIdent(ident, is_raw) => {
468-
Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
469-
}
470-
NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
471-
_ => Cow::Borrowed(self),
472-
},
473-
_ => Cow::Borrowed(self),
474-
}
475-
}
476-
477453
/// Returns an identifier if this token is an identifier.
478454
pub fn ident(&self) -> Option<(Ident, /* is_raw */ bool)> {
479-
let token = self.uninterpolate();
480-
match token.kind {
481-
Ident(name, is_raw) => Some((Ident::new(name, token.span), is_raw)),
455+
match self.kind {
456+
Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
482457
_ => None,
483458
}
484459
}
485460

486461
/// Returns a lifetime identifier if this token is a lifetime.
487462
pub fn lifetime(&self) -> Option<Ident> {
488-
let token = self.uninterpolate();
489-
match token.kind {
490-
Lifetime(name) => Some(Ident::new(name, token.span)),
463+
match self.kind {
464+
Lifetime(name) => Some(Ident::new(name, self.span)),
491465
_ => None,
492466
}
493467
}
@@ -521,7 +495,7 @@ impl Token {
521495
/// (which happens while parsing the result of macro expansion)?
522496
pub fn is_whole_expr(&self) -> bool {
523497
if let Interpolated(ref nt) = self.kind
524-
&& let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt
498+
&& let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = **nt
525499
{
526500
return true;
527501
}
@@ -679,8 +653,6 @@ pub enum Nonterminal {
679653
NtPat(P<ast::Pat>),
680654
NtExpr(P<ast::Expr>),
681655
NtTy(P<ast::Ty>),
682-
NtIdent(Ident, /* is_raw */ bool),
683-
NtLifetime(Ident),
684656
NtLiteral(P<ast::Expr>),
685657
/// Stuff inside brackets for attributes
686658
NtMeta(P<ast::AttrItem>),
@@ -779,7 +751,6 @@ impl Nonterminal {
779751
NtPat(pat) => pat.span,
780752
NtExpr(expr) | NtLiteral(expr) => expr.span,
781753
NtTy(ty) => ty.span,
782-
NtIdent(ident, _) | NtLifetime(ident) => ident.span,
783754
NtMeta(attr_item) => attr_item.span(),
784755
NtPath(path) => path.span,
785756
NtVis(vis) => vis.span,
@@ -790,11 +761,7 @@ impl Nonterminal {
790761
impl PartialEq for Nonterminal {
791762
fn eq(&self, rhs: &Self) -> bool {
792763
match (self, rhs) {
793-
(NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => {
794-
ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
795-
}
796-
(NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
797-
// FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
764+
// FIXME: Assume that all nonterminals are not equal, because we can't compare them
798765
// correctly based on data from AST. This will prevent them from matching each other
799766
// in macros. The comparison will become possible only when each nonterminal has an
800767
// attached token stream from which it was parsed.
@@ -812,12 +779,10 @@ impl fmt::Debug for Nonterminal {
812779
NtPat(..) => f.pad("NtPat(..)"),
813780
NtExpr(..) => f.pad("NtExpr(..)"),
814781
NtTy(..) => f.pad("NtTy(..)"),
815-
NtIdent(..) => f.pad("NtIdent(..)"),
816782
NtLiteral(..) => f.pad("NtLiteral(..)"),
817783
NtMeta(..) => f.pad("NtMeta(..)"),
818784
NtPath(..) => f.pad("NtPath(..)"),
819785
NtVis(..) => f.pad("NtVis(..)"),
820-
NtLifetime(..) => f.pad("NtLifetime(..)"),
821786
}
822787
}
823788
}

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -93,13 +93,6 @@ impl TokenTree {
9393
pub fn token(kind: TokenKind, span: Span) -> TokenTree {
9494
TokenTree::Token(Token::new(kind, span))
9595
}
96-
97-
pub fn uninterpolate(self) -> TokenTree {
98-
match self {
99-
TokenTree::Token(token) => TokenTree::Token(token.uninterpolate().into_owned()),
100-
tt => tt,
101-
}
102-
}
10396
}
10497

10598
impl<CTX> HashStable<CTX> for TokenStream

‎compiler/rustc_ast/src/util/literal.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ impl Lit {
216216
///
217217
/// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation.
218218
pub fn from_token(token: &Token) -> Result<Lit, LitError> {
219-
let lit = match token.uninterpolate().kind {
219+
let lit = match token.kind {
220220
token::Ident(name, false) if name.is_bool_lit() => {
221221
token::Lit::new(token::Bool, name, None)
222222
}

‎compiler/rustc_ast_pretty/src/pprust/state.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -713,8 +713,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
713713
token::NtBlock(ref e) => self.block_to_string(e),
714714
token::NtStmt(ref e) => self.stmt_to_string(e),
715715
token::NtPat(ref e) => self.pat_to_string(e),
716-
token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw).to_string(),
717-
token::NtLifetime(e) => e.to_string(),
718716
token::NtLiteral(ref e) => self.expr_to_string(e),
719717
token::NtVis(ref e) => self.vis_to_string(e),
720718
}

‎compiler/rustc_builtin_macros/src/asm.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -502,7 +502,7 @@ fn parse_reg<'a>(
502502
explicit_reg: &mut bool,
503503
) -> PResult<'a, ast::InlineAsmRegOrRegClass> {
504504
p.expect(&token::OpenDelim(Delimiter::Parenthesis))?;
505-
let result = match p.token.uninterpolate().kind {
505+
let result = match p.token.kind {
506506
token::Ident(name, false) => ast::InlineAsmRegOrRegClass::RegClass(name),
507507
token::Literal(token::Lit { kind: token::LitKind::Str, symbol, suffix: _ }) => {
508508
*explicit_reg = true;

‎compiler/rustc_expand/src/mbe/macro_parser.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,9 @@ crate enum NamedMatch {
344344
// A metavar match of type `tt`.
345345
MatchedTokenTree(rustc_ast::tokenstream::TokenTree),
346346

347+
// njn: comment
348+
MatchedToken(token::Token),
349+
347350
// A metavar match of any type other than `tt`.
348351
MatchedNonterminal(Lrc<Nonterminal>),
349352
}
@@ -624,6 +627,7 @@ impl TtParser {
624627
let m = match nt {
625628
NtOrTt::Nt(nt) => MatchedNonterminal(Lrc::new(nt)),
626629
NtOrTt::Tt(tt) => MatchedTokenTree(tt),
630+
NtOrTt::Token(token) => MatchedToken(token),
627631
};
628632
mp.push_match(next_metavar, seq_depth, m);
629633
mp.idx += 1;

‎compiler/rustc_expand/src/mbe/transcribe.rs

Lines changed: 25 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use crate::base::ExtCtxt;
2-
use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, MatchedTokenTree, NamedMatch};
2+
use crate::mbe::macro_parser::{NamedMatch, NamedMatch::*};
33
use crate::mbe::{self, MetaVarExpr};
44
use rustc_ast::mut_visit::{self, MutVisitor};
55
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
@@ -226,16 +226,22 @@ pub(super) fn transcribe<'a>(
226226
MatchedTokenTree(ref tt) => {
227227
// `tt`s are emitted into the output stream directly as "raw tokens",
228228
// without wrapping them into groups.
229-
let token = tt.clone();
230-
result.push(token.into());
229+
let tt = tt.clone();
230+
result.push(tt.into());
231+
}
232+
MatchedToken(ref token) => {
233+
// njn: this doesn't quite work, but adding
234+
// visit_token makes things worse
235+
let tt = TokenTree::Token(token.clone());
236+
result.push(tt.into());
231237
}
232238
MatchedNonterminal(ref nt) => {
233239
// Other variables are emitted into the output stream as groups with
234240
// `Delimiter::Invisible` to maintain parsing priorities.
235241
// `Interpolated` is currently used for such groups in rustc parser.
236242
marker.visit_span(&mut sp);
237-
let token = TokenTree::token(token::Interpolated(nt.clone()), sp);
238-
result.push(token.into());
243+
let tt = TokenTree::token(token::Interpolated(nt.clone()), sp);
244+
result.push(tt.into());
239245
}
240246
MatchedSeq(..) => {
241247
// We were unable to descend far enough. This is an error.
@@ -306,8 +312,8 @@ fn lookup_cur_matched<'a>(
306312
let mut matched = matched;
307313
for &(idx, _) in repeats {
308314
match matched {
309-
MatchedTokenTree(_) | MatchedNonterminal(_) => break,
310315
MatchedSeq(ref ads) => matched = ads.get(idx).unwrap(),
316+
_ => break,
311317
}
312318
}
313319

@@ -396,8 +402,8 @@ fn lockstep_iter_size(
396402
let name = MacroRulesNormalizedIdent::new(name);
397403
match lookup_cur_matched(name, interpolations, repeats) {
398404
Some(matched) => match matched {
399-
MatchedTokenTree(_) | MatchedNonterminal(_) => LockstepIterSize::Unconstrained,
400405
MatchedSeq(ref ads) => LockstepIterSize::Constraint(ads.len(), name),
406+
_ => LockstepIterSize::Unconstrained,
401407
},
402408
_ => LockstepIterSize::Unconstrained,
403409
}
@@ -443,18 +449,6 @@ fn count_repetitions<'a>(
443449
sp: &DelimSpan,
444450
) -> PResult<'a, usize> {
445451
match matched {
446-
MatchedTokenTree(_) | MatchedNonterminal(_) => {
447-
if declared_lhs_depth == 0 {
448-
return Err(cx.struct_span_err(
449-
sp.entire(),
450-
"`count` can not be placed inside the inner-most repetition",
451-
));
452-
}
453-
match depth_opt {
454-
None => Ok(1),
455-
Some(_) => Err(out_of_bounds_err(cx, declared_lhs_depth, sp.entire(), "count")),
456-
}
457-
}
458452
MatchedSeq(ref named_matches) => {
459453
let new_declared_lhs_depth = declared_lhs_depth + 1;
460454
match depth_opt {
@@ -469,6 +463,18 @@ fn count_repetitions<'a>(
469463
.sum(),
470464
}
471465
}
466+
_ => {
467+
if declared_lhs_depth == 0 {
468+
return Err(cx.struct_span_err(
469+
sp.entire(),
470+
"`count` can not be placed inside the inner-most repetition",
471+
));
472+
}
473+
match depth_opt {
474+
None => Ok(1),
475+
Some(_) => Err(out_of_bounds_err(cx, declared_lhs_depth, sp.entire(), "count")),
476+
}
477+
}
472478
}
473479
}
474480
// `repeats` records all of the nested levels at which we are currently

‎compiler/rustc_expand/src/proc_macro_server.rs

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -175,9 +175,6 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec<Self>, &mut Rustc<'_, '_>)>
175175
tt!(Punct::new('#', false))
176176
}
177177

178-
Interpolated(nt) if let NtIdent(ident, is_raw) = *nt => {
179-
TokenTree::Ident(Ident::new(rustc.sess(), ident.name, is_raw, ident.span))
180-
}
181178
Interpolated(nt) => {
182179
let stream = nt_to_tokenstream(&nt, rustc.sess(), CanSynthesizeMissingTokens::No);
183180
TokenTree::Group(Group {

‎compiler/rustc_parse/src/lib.rs

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -281,12 +281,6 @@ pub fn nt_to_tokenstream(
281281
Nonterminal::NtStmt(ref stmt) => prepend_attrs(&stmt.attrs(), stmt.tokens()),
282282
Nonterminal::NtPat(ref pat) => convert_tokens(pat.tokens.as_ref()),
283283
Nonterminal::NtTy(ref ty) => convert_tokens(ty.tokens.as_ref()),
284-
Nonterminal::NtIdent(ident, is_raw) => {
285-
Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into())
286-
}
287-
Nonterminal::NtLifetime(ident) => {
288-
Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into())
289-
}
290284
Nonterminal::NtMeta(ref attr) => convert_tokens(attr.tokens.as_ref()),
291285
Nonterminal::NtPath(ref path) => convert_tokens(path.tokens.as_ref()),
292286
Nonterminal::NtVis(ref vis) => convert_tokens(vis.tokens.as_ref()),

‎compiler/rustc_parse/src/parser/attr_wrapper.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -229,8 +229,8 @@ impl<'a> Parser<'a> {
229229
let (mut ret, trailing) = ret?;
230230

231231
// When we're not in `capture-cfg` mode, then bail out early if:
232-
// 1. Our target doesn't support tokens at all (e.g we're parsing an `NtIdent`)
233-
// so there's nothing for us to do.
232+
// 1. Our target doesn't support tokens at all so there's nothing for
233+
// us to do.
234234
// 2. Our target already has tokens set (e.g. we've parsed something
235235
// like `#[my_attr] $item`. The actual parsing code takes care of prepending
236236
// any attributes to the nonterminal, so we don't need to modify the

0 commit comments

Comments
 (0)