Skip to content

Commit 2f5db75

Browse files
borsbors
committed
Auto merge of rust-lang#138478 - nnethercote:rm-NtExpr-NtLiteral, r=<try>
Remove `NtExpr` and `NtLiteral` The next part of rust-lang#124141. r? `@petrochenkov`
2 parents ae8ab87 + 7f6b06c commit 2f5db75

31 files changed

+832
-586
lines changed

‎compiler/rustc_ast/src/ast_traits.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -209,13 +209,11 @@ impl HasTokens for Attribute {
209209
impl HasTokens for Nonterminal {
210210
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
211211
match self {
212-
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
213212
Nonterminal::NtBlock(block) => block.tokens(),
214213
}
215214
}
216215
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
217216
match self {
218-
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
219217
Nonterminal::NtBlock(block) => block.tokens_mut(),
220218
}
221219
}

‎compiler/rustc_ast/src/mut_visit.rs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -900,8 +900,6 @@ pub fn visit_token<T: MutVisitor>(vis: &mut T, t: &mut Token) {
900900
fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
901901
match nt {
902902
token::NtBlock(block) => vis.visit_block(block),
903-
token::NtExpr(expr) => vis.visit_expr(expr),
904-
token::NtLiteral(expr) => vis.visit_expr(expr),
905903
}
906904
}
907905

‎compiler/rustc_ast/src/token.rs

Lines changed: 23 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -198,16 +198,17 @@ impl Lit {
198198
}
199199
}
200200

201-
/// Keep this in sync with `Token::can_begin_literal_maybe_minus` excluding unary negation.
201+
/// Keep this in sync with `Token::can_begin_literal_maybe_minus` and
202+
/// `Parser::eat_token_lit` (excluding unary negation).
202203
pub fn from_token(token: &Token) -> Option<Lit> {
203204
match token.uninterpolate().kind {
204205
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
205206
Literal(token_lit) => Some(token_lit),
206-
Interpolated(ref nt)
207-
if let NtExpr(expr) | NtLiteral(expr) = &**nt
208-
&& let ast::ExprKind::Lit(token_lit) = expr.kind =>
209-
{
210-
Some(token_lit)
207+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
208+
MetaVarKind::Literal | MetaVarKind::Expr { .. },
209+
))) => {
210+
// Unreachable with the current test suite.
211+
panic!("from_token metavar");
211212
}
212213
_ => None,
213214
}
@@ -590,6 +591,9 @@ impl Token {
590591
/// for which spans affect name resolution and edition checks.
591592
/// Note that keywords are also identifiers, so they should use this
592593
/// if they keep spans or perform edition checks.
594+
//
595+
// Note: `Parser::uninterpolated_token_span` may give better information
596+
// than this method does.
593597
pub fn uninterpolated_span(&self) -> Span {
594598
match self.kind {
595599
NtIdent(ident, _) | NtLifetime(ident, _) => ident.span,
@@ -642,12 +646,7 @@ impl Token {
642646
PathSep | // global path
643647
Lifetime(..) | // labeled loop
644648
Pound => true, // expression attributes
645-
Interpolated(ref nt) =>
646-
matches!(&**nt,
647-
NtBlock(..) |
648-
NtExpr(..) |
649-
NtLiteral(..)
650-
),
649+
Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
651650
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
652651
MetaVarKind::Block |
653652
MetaVarKind::Expr { .. } |
@@ -677,11 +676,6 @@ impl Token {
677676
Lt | // path (UFCS constant)
678677
Shl => true, // path (double UFCS)
679678
Or => matches!(pat_kind, PatWithOr), // leading vert `|` or-pattern
680-
Interpolated(nt) =>
681-
matches!(&**nt,
682-
| NtExpr(..)
683-
| NtLiteral(..)
684-
),
685679
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
686680
MetaVarKind::Expr { .. } |
687681
MetaVarKind::Literal |
@@ -724,7 +718,7 @@ impl Token {
724718
match self.kind {
725719
OpenDelim(Delimiter::Brace) | Literal(..) | Minus => true,
726720
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
727-
Interpolated(ref nt) => matches!(&**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
721+
Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
728722
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
729723
MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal,
730724
))) => true,
@@ -768,22 +762,12 @@ impl Token {
768762
///
769763
/// In other words, would this token be a valid start of `parse_literal_maybe_minus`?
770764
///
771-
/// Keep this in sync with and `Lit::from_token`, excluding unary negation.
765+
/// Keep this in sync with `Lit::from_token` and `Parser::eat_token_lit`
766+
/// (excluding unary negation).
772767
pub fn can_begin_literal_maybe_minus(&self) -> bool {
773768
match self.uninterpolate().kind {
774769
Literal(..) | Minus => true,
775770
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
776-
Interpolated(ref nt) => match &**nt {
777-
NtLiteral(_) => true,
778-
NtExpr(e) => match &e.kind {
779-
ast::ExprKind::Lit(_) => true,
780-
ast::ExprKind::Unary(ast::UnOp::Neg, e) => {
781-
matches!(&e.kind, ast::ExprKind::Lit(_))
782-
}
783-
_ => false,
784-
},
785-
_ => false,
786-
},
787771
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
788772
MetaVarKind::Literal => true,
789773
MetaVarKind::Expr { can_begin_literal_maybe_minus, .. } => {
@@ -798,14 +782,6 @@ impl Token {
798782
pub fn can_begin_string_literal(&self) -> bool {
799783
match self.uninterpolate().kind {
800784
Literal(..) => true,
801-
Interpolated(ref nt) => match &**nt {
802-
NtLiteral(_) => true,
803-
NtExpr(e) => match &e.kind {
804-
ast::ExprKind::Lit(_) => true,
805-
_ => false,
806-
},
807-
_ => false,
808-
},
809785
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
810786
MetaVarKind::Literal => true,
811787
MetaVarKind::Expr { can_begin_string_literal, .. } => can_begin_string_literal,
@@ -869,19 +845,25 @@ impl Token {
869845

870846
/// Is this a pre-parsed expression dropped into the token stream
871847
/// (which happens while parsing the result of macro expansion)?
872-
pub fn is_whole_expr(&self) -> bool {
848+
pub fn is_metavar_expr(&self) -> bool {
873849
#[allow(irrefutable_let_patterns)] // FIXME: temporary
874850
if let Interpolated(nt) = &self.kind
875-
&& let NtExpr(_) | NtLiteral(_) | NtBlock(_) = &**nt
851+
&& let NtBlock(_) = &**nt
876852
{
877853
true
854+
} else if matches!(
855+
self.is_metavar_seq(),
856+
Some(MetaVarKind::Expr { .. } | MetaVarKind::Literal | MetaVarKind::Path)
857+
) {
858+
true
878859
} else {
879860
matches!(self.is_metavar_seq(), Some(MetaVarKind::Path))
880861
}
881862
}
882863

883864
/// Is the token an interpolated block (`$b:block`)?
884865
pub fn is_whole_block(&self) -> bool {
866+
#[allow(irrefutable_let_patterns)] // FIXME: temporary
885867
if let Interpolated(nt) = &self.kind
886868
&& let NtBlock(..) = &**nt
887869
{
@@ -1105,8 +1087,6 @@ pub enum NtExprKind {
11051087
/// For interpolation during macro expansion.
11061088
pub enum Nonterminal {
11071089
NtBlock(P<ast::Block>),
1108-
NtExpr(P<ast::Expr>),
1109-
NtLiteral(P<ast::Expr>),
11101090
}
11111091

11121092
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
@@ -1196,15 +1176,12 @@ impl Nonterminal {
11961176
pub fn use_span(&self) -> Span {
11971177
match self {
11981178
NtBlock(block) => block.span,
1199-
NtExpr(expr) | NtLiteral(expr) => expr.span,
12001179
}
12011180
}
12021181

12031182
pub fn descr(&self) -> &'static str {
12041183
match self {
12051184
NtBlock(..) => "block",
1206-
NtExpr(..) => "expression",
1207-
NtLiteral(..) => "literal",
12081185
}
12091186
}
12101187
}
@@ -1223,8 +1200,6 @@ impl fmt::Debug for Nonterminal {
12231200
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
12241201
match *self {
12251202
NtBlock(..) => f.pad("NtBlock(..)"),
1226-
NtExpr(..) => f.pad("NtExpr(..)"),
1227-
NtLiteral(..) => f.pad("NtLiteral(..)"),
12281203
}
12291204
}
12301205
}
@@ -1247,7 +1222,7 @@ mod size_asserts {
12471222
// tidy-alphabetical-start
12481223
static_assert_size!(Lit, 12);
12491224
static_assert_size!(LitKind, 2);
1250-
static_assert_size!(Nonterminal, 16);
1225+
static_assert_size!(Nonterminal, 8);
12511226
static_assert_size!(Token, 24);
12521227
static_assert_size!(TokenKind, 16);
12531228
// tidy-alphabetical-end

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 39 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -233,35 +233,52 @@ fn attrs_and_tokens_to_token_trees(
233233

234234
// Insert inner attribute tokens.
235235
if !inner_attrs.is_empty() {
236-
let mut found = false;
237-
// Check the last two trees (to account for a trailing semi)
238-
for tree in res.iter_mut().rev().take(2) {
239-
if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree {
240-
// Inner attributes are only supported on extern blocks, functions,
241-
// impls, and modules. All of these have their inner attributes
242-
// placed at the beginning of the rightmost outermost braced group:
243-
// e.g. fn foo() { #![my_attr] }
244-
//
245-
// Therefore, we can insert them back into the right location
246-
// without needing to do any extra position tracking.
247-
//
248-
// Note: Outline modules are an exception - they can
249-
// have attributes like `#![my_attr]` at the start of a file.
250-
// Support for custom attributes in this position is not
251-
// properly implemented - we always synthesize fake tokens,
252-
// so we never reach this code.
236+
let found = insert_inner_attrs(inner_attrs, res);
237+
assert!(found, "Failed to find trailing delimited group in: {res:?}");
238+
}
239+
240+
// Inner attributes are only supported on blocks, functions, impls, and
241+
// modules. All of these have their inner attributes placed at the
242+
// beginning of the rightmost outermost braced group:
243+
// e.g. `fn foo() { #![my_attr] }`. (Note: the braces may be within
244+
// invisible delimiters.)
245+
//
246+
// Therefore, we can insert them back into the right location without
247+
// needing to do any extra position tracking.
248+
//
249+
// Note: Outline modules are an exception - they can have attributes like
250+
// `#![my_attr]` at the start of a file. Support for custom attributes in
251+
// this position is not properly implemented - we always synthesize fake
252+
// tokens, so we never reach this code.
253+
fn insert_inner_attrs(inner_attrs: &[Attribute], tts: &mut Vec<TokenTree>) -> bool {
254+
for tree in tts.iter_mut().rev() {
255+
if let TokenTree::Delimited(span, spacing, Delimiter::Brace, stream) = tree {
256+
// Found it: the rightmost, outermost braced group.
253257
let mut tts = vec![];
254258
for inner_attr in inner_attrs {
255259
tts.extend(inner_attr.token_trees());
256260
}
257-
tts.extend(delim_tokens.0.iter().cloned());
261+
tts.extend(stream.0.iter().cloned());
258262
let stream = TokenStream::new(tts);
259-
*tree = TokenTree::Delimited(*span, *spacing, *delim, stream);
260-
found = true;
261-
break;
263+
*tree = TokenTree::Delimited(*span, *spacing, Delimiter::Brace, stream);
264+
return true;
265+
} else if let TokenTree::Delimited(span, spacing, Delimiter::Invisible(src), stream) =
266+
tree
267+
{
268+
// Recurse inside invisible delimiters.
269+
let mut vec: Vec<_> = stream.iter().cloned().collect();
270+
if insert_inner_attrs(inner_attrs, &mut vec) {
271+
*tree = TokenTree::Delimited(
272+
*span,
273+
*spacing,
274+
Delimiter::Invisible(*src),
275+
TokenStream::new(vec),
276+
);
277+
return true;
278+
}
262279
}
263280
}
264-
assert!(found, "Failed to find trailing delimited group in: {res:?}");
281+
false
265282
}
266283
}
267284

@@ -462,7 +479,6 @@ impl TokenStream {
462479
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
463480
match nt {
464481
Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
465-
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
466482
}
467483
}
468484

‎compiler/rustc_expand/src/mbe/transcribe.rs

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,10 @@ use std::sync::Arc;
33

44
use rustc_ast::mut_visit::{self, MutVisitor};
55
use rustc_ast::token::{
6-
self, Delimiter, IdentIsRaw, InvisibleOrigin, Lit, LitKind, MetaVarKind, Nonterminal, Token,
7-
TokenKind,
6+
self, Delimiter, IdentIsRaw, InvisibleOrigin, Lit, LitKind, MetaVarKind, Token, TokenKind,
87
};
98
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
10-
use rustc_ast::{ExprKind, StmtKind, TyKind};
9+
use rustc_ast::{ExprKind, StmtKind, TyKind, UnOp};
1110
use rustc_data_structures::fx::FxHashMap;
1211
use rustc_errors::{Diag, DiagCtxtHandle, PResult, pluralize};
1312
use rustc_parse::lexer::nfc_normalize;
@@ -340,6 +339,30 @@ pub(super) fn transcribe<'a>(
340339
MetaVarKind::Pat(*pat_kind),
341340
TokenStream::from_ast(pat),
342341
),
342+
MatchedSingle(ParseNtResult::Expr(expr, kind)) => {
343+
let (can_begin_literal_maybe_minus, can_begin_string_literal) =
344+
match &expr.kind {
345+
ExprKind::Lit(_) => (true, true),
346+
ExprKind::Unary(UnOp::Neg, e)
347+
if matches!(&e.kind, ExprKind::Lit(_)) =>
348+
{
349+
(true, false)
350+
}
351+
_ => (false, false),
352+
};
353+
mk_delimited(
354+
expr.span,
355+
MetaVarKind::Expr {
356+
kind: *kind,
357+
can_begin_literal_maybe_minus,
358+
can_begin_string_literal,
359+
},
360+
TokenStream::from_ast(expr),
361+
)
362+
}
363+
MatchedSingle(ParseNtResult::Literal(lit)) => {
364+
mk_delimited(lit.span, MetaVarKind::Literal, TokenStream::from_ast(lit))
365+
}
343366
MatchedSingle(ParseNtResult::Ty(ty)) => {
344367
let is_path = matches!(&ty.kind, TyKind::Path(None, _path));
345368
mk_delimited(
@@ -869,10 +892,8 @@ fn extract_symbol_from_pnr<'a>(
869892
},
870893
_,
871894
)) => Ok(*symbol),
872-
ParseNtResult::Nt(nt)
873-
if let Nonterminal::NtLiteral(expr) = &**nt
874-
&& let ExprKind::Lit(Lit { kind: LitKind::Str, symbol, suffix: None }) =
875-
&expr.kind =>
895+
ParseNtResult::Literal(expr)
896+
if let ExprKind::Lit(Lit { kind: LitKind::Str, symbol, suffix: None }) = &expr.kind =>
876897
{
877898
Ok(*symbol)
878899
}

‎compiler/rustc_parse/messages.ftl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -864,7 +864,7 @@ parse_unexpected_parentheses_in_match_arm_pattern = unexpected parentheses surro
864864
parse_unexpected_self_in_generic_parameters = unexpected keyword `Self` in generic parameters
865865
.note = you cannot use `Self` as a generic parameter because it is reserved for associated items
866866
867-
parse_unexpected_token_after_dot = unexpected token: `{$actual}`
867+
parse_unexpected_token_after_dot = unexpected token: {$actual}
868868
869869
parse_unexpected_token_after_label = expected `while`, `for`, `loop` or `{"{"}` after a label
870870
.suggestion_remove_label = consider removing the label

‎compiler/rustc_parse/src/errors.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1714,10 +1714,10 @@ pub(crate) struct SelfArgumentPointer {
17141714

17151715
#[derive(Diagnostic)]
17161716
#[diag(parse_unexpected_token_after_dot)]
1717-
pub(crate) struct UnexpectedTokenAfterDot<'a> {
1717+
pub(crate) struct UnexpectedTokenAfterDot {
17181718
#[primary_span]
17191719
pub span: Span,
1720-
pub actual: Cow<'a, str>,
1720+
pub actual: String,
17211721
}
17221722

17231723
#[derive(Diagnostic)]

0 commit comments

Comments
 (0)