Skip to content

Commit 0699d99

Browse files
borsbors
committed
Auto merge of #114115 - nnethercote:less-token-tree-cloning, r=petrochenkov
Less `TokenTree` cloning `TokenTreeCursor` has this comment on it: ``` // FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones. ``` This PR completes that FIXME. It doesn't have much perf effect, but at least we now know that. r? `@petrochenkov`
2 parents 6cacb52 + 4ebf2be commit 0699d99

File tree

9 files changed

+83
-90
lines changed

9 files changed

+83
-90
lines changed

‎compiler/rustc_ast/src/attr/mod.rs

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -285,17 +285,17 @@ impl MetaItem {
285285
self.kind.value_str()
286286
}
287287

288-
fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
288+
fn from_tokens<'a, I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
289289
where
290-
I: Iterator<Item = TokenTree>,
290+
I: Iterator<Item = &'a TokenTree>,
291291
{
292292
// FIXME: Share code with `parse_path`.
293-
let path = match tokens.next().map(TokenTree::uninterpolate) {
294-
Some(TokenTree::Token(
295-
Token { kind: kind @ (token::Ident(..) | token::ModSep), span },
293+
let path = match tokens.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref() {
294+
Some(&TokenTree::Token(
295+
Token { kind: ref kind @ (token::Ident(..) | token::ModSep), span },
296296
_,
297297
)) => 'arm: {
298-
let mut segments = if let token::Ident(name, _) = kind {
298+
let mut segments = if let &token::Ident(name, _) = kind {
299299
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) =
300300
tokens.peek()
301301
{
@@ -308,8 +308,8 @@ impl MetaItem {
308308
thin_vec![PathSegment::path_root(span)]
309309
};
310310
loop {
311-
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) =
312-
tokens.next().map(TokenTree::uninterpolate)
311+
if let Some(&TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) =
312+
tokens.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref()
313313
{
314314
segments.push(PathSegment::from_ident(Ident::new(name, span)));
315315
} else {
@@ -326,7 +326,7 @@ impl MetaItem {
326326
let span = span.with_hi(segments.last().unwrap().ident.span.hi());
327327
Path { span, segments, tokens: None }
328328
}
329-
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &*nt {
329+
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &**nt {
330330
token::Nonterminal::NtMeta(item) => return item.meta(item.path.span),
331331
token::Nonterminal::NtPath(path) => (**path).clone(),
332332
_ => return None,
@@ -354,7 +354,7 @@ impl MetaItemKind {
354354
}
355355

356356
fn list_from_tokens(tokens: TokenStream) -> Option<ThinVec<NestedMetaItem>> {
357-
let mut tokens = tokens.into_trees().peekable();
357+
let mut tokens = tokens.trees().peekable();
358358
let mut result = ThinVec::new();
359359
while tokens.peek().is_some() {
360360
let item = NestedMetaItem::from_tokens(&mut tokens)?;
@@ -367,12 +367,12 @@ impl MetaItemKind {
367367
Some(result)
368368
}
369369

370-
fn name_value_from_tokens(
371-
tokens: &mut impl Iterator<Item = TokenTree>,
370+
fn name_value_from_tokens<'a>(
371+
tokens: &mut impl Iterator<Item = &'a TokenTree>,
372372
) -> Option<MetaItemKind> {
373373
match tokens.next() {
374374
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
375-
MetaItemKind::name_value_from_tokens(&mut inner_tokens.into_trees())
375+
MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
376376
}
377377
Some(TokenTree::Token(token, _)) => {
378378
MetaItemLit::from_token(&token).map(MetaItemKind::NameValue)
@@ -381,8 +381,8 @@ impl MetaItemKind {
381381
}
382382
}
383383

384-
fn from_tokens(
385-
tokens: &mut iter::Peekable<impl Iterator<Item = TokenTree>>,
384+
fn from_tokens<'a>(
385+
tokens: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
386386
) -> Option<MetaItemKind> {
387387
match tokens.peek() {
388388
Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => {
@@ -501,9 +501,9 @@ impl NestedMetaItem {
501501
self.meta_item().is_some()
502502
}
503503

504-
fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<NestedMetaItem>
504+
fn from_tokens<'a, I>(tokens: &mut iter::Peekable<I>) -> Option<NestedMetaItem>
505505
where
506-
I: Iterator<Item = TokenTree>,
506+
I: Iterator<Item = &'a TokenTree>,
507507
{
508508
match tokens.peek() {
509509
Some(TokenTree::Token(token, _))
@@ -513,9 +513,8 @@ impl NestedMetaItem {
513513
return Some(NestedMetaItem::Lit(lit));
514514
}
515515
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
516-
let inner_tokens = inner_tokens.clone();
517516
tokens.next();
518-
return NestedMetaItem::from_tokens(&mut inner_tokens.into_trees().peekable());
517+
return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
519518
}
520519
_ => {}
521520
}

‎compiler/rustc_ast/src/tokenstream.rs

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
2525
use rustc_span::{Span, DUMMY_SP};
2626
use smallvec::{smallvec, SmallVec};
2727

28+
use std::borrow::Cow;
2829
use std::{fmt, iter, mem};
2930

3031
/// When the main Rust parser encounters a syntax-extension invocation, it
@@ -98,12 +99,13 @@ impl TokenTree {
9899
TokenTree::Token(Token::new(kind, span), Spacing::Joint)
99100
}
100101

101-
pub fn uninterpolate(self) -> TokenTree {
102+
pub fn uninterpolate(&self) -> Cow<'_, TokenTree> {
102103
match self {
103-
TokenTree::Token(token, spacing) => {
104-
TokenTree::Token(token.uninterpolate().into_owned(), spacing)
105-
}
106-
tt => tt,
104+
TokenTree::Token(token, spacing) => match token.uninterpolate() {
105+
Cow::Owned(token) => Cow::Owned(TokenTree::Token(token, *spacing)),
106+
Cow::Borrowed(_) => Cow::Borrowed(self),
107+
},
108+
_ => Cow::Borrowed(self),
107109
}
108110
}
109111
}
@@ -595,26 +597,21 @@ impl<'t> Iterator for RefTokenTreeCursor<'t> {
595597
}
596598
}
597599

598-
/// Owning by-value iterator over a [`TokenStream`], that produces `TokenTree`
600+
/// Owning by-value iterator over a [`TokenStream`], that produces `&TokenTree`
599601
/// items.
600-
// FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones.
602+
///
603+
/// Doesn't impl `Iterator` because Rust doesn't permit an owning iterator to
604+
/// return `&T` from `next`; the need for an explicit lifetime in the `Item`
605+
/// associated type gets in the way. Instead, use `next_ref` (which doesn't
606+
/// involve associated types) for getting individual elements, or
607+
/// `RefTokenTreeCursor` if you really want an `Iterator`, e.g. in a `for`
608+
/// loop.
601609
#[derive(Clone)]
602610
pub struct TokenTreeCursor {
603611
pub stream: TokenStream,
604612
index: usize,
605613
}
606614

607-
impl Iterator for TokenTreeCursor {
608-
type Item = TokenTree;
609-
610-
fn next(&mut self) -> Option<TokenTree> {
611-
self.stream.0.get(self.index).map(|tree| {
612-
self.index += 1;
613-
tree.clone()
614-
})
615-
}
616-
}
617-
618615
impl TokenTreeCursor {
619616
fn new(stream: TokenStream) -> Self {
620617
TokenTreeCursor { stream, index: 0 }

‎compiler/rustc_expand/src/config.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -365,9 +365,9 @@ impl<'a> StripUnconfigured<'a> {
365365

366366
// Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token
367367
// for `attr` when we expand it to `#[attr]`
368-
let mut orig_trees = orig_tokens.into_trees();
368+
let mut orig_trees = orig_tokens.trees();
369369
let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }, _) =
370-
orig_trees.next().unwrap()
370+
orig_trees.next().unwrap().clone()
371371
else {
372372
panic!("Bad tokens for attribute {:?}", attr);
373373
};
@@ -377,7 +377,7 @@ impl<'a> StripUnconfigured<'a> {
377377
if attr.style == AttrStyle::Inner {
378378
// For inner attributes, we do the same thing for the `!` in `#![some_attr]`
379379
let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) =
380-
orig_trees.next().unwrap()
380+
orig_trees.next().unwrap().clone()
381381
else {
382382
panic!("Bad tokens for attribute {:?}", attr);
383383
};

‎compiler/rustc_expand/src/mbe/macro_rules.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -500,7 +500,7 @@ pub fn compile_declarative_macro(
500500
.map(|m| {
501501
if let MatchedTokenTree(tt) = m {
502502
let tt = mbe::quoted::parse(
503-
TokenStream::new(vec![tt.clone()]),
503+
&TokenStream::new(vec![tt.clone()]),
504504
true,
505505
&sess.parse_sess,
506506
def.id,
@@ -524,7 +524,7 @@ pub fn compile_declarative_macro(
524524
.map(|m| {
525525
if let MatchedTokenTree(tt) = m {
526526
return mbe::quoted::parse(
527-
TokenStream::new(vec![tt.clone()]),
527+
&TokenStream::new(vec![tt.clone()]),
528528
false,
529529
&sess.parse_sess,
530530
def.id,

‎compiler/rustc_expand/src/mbe/quoted.rs

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \
3636
///
3737
/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`.
3838
pub(super) fn parse(
39-
input: tokenstream::TokenStream,
39+
input: &tokenstream::TokenStream,
4040
parsing_patterns: bool,
4141
sess: &ParseSess,
4242
node_id: NodeId,
@@ -48,15 +48,15 @@ pub(super) fn parse(
4848

4949
// For each token tree in `input`, parse the token into a `self::TokenTree`, consuming
5050
// additional trees if need be.
51-
let mut trees = input.into_trees();
51+
let mut trees = input.trees();
5252
while let Some(tree) = trees.next() {
5353
// Given the parsed tree, if there is a metavar and we are expecting matchers, actually
5454
// parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`).
5555
let tree = parse_tree(tree, &mut trees, parsing_patterns, sess, node_id, features, edition);
5656
match tree {
5757
TokenTree::MetaVar(start_sp, ident) if parsing_patterns => {
5858
let span = match trees.next() {
59-
Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span }, _)) => {
59+
Some(&tokenstream::TokenTree::Token(Token { kind: token::Colon, span }, _)) => {
6060
match trees.next() {
6161
Some(tokenstream::TokenTree::Token(token, _)) => match token.ident() {
6262
Some((frag, _)) => {
@@ -96,10 +96,10 @@ pub(super) fn parse(
9696
}
9797
_ => token.span,
9898
},
99-
tree => tree.as_ref().map_or(span, tokenstream::TokenTree::span),
99+
tree => tree.map_or(span, tokenstream::TokenTree::span),
100100
}
101101
}
102-
tree => tree.as_ref().map_or(start_sp, tokenstream::TokenTree::span),
102+
tree => tree.map_or(start_sp, tokenstream::TokenTree::span),
103103
};
104104

105105
result.push(TokenTree::MetaVarDecl(span, ident, None));
@@ -134,9 +134,9 @@ fn maybe_emit_macro_metavar_expr_feature(features: &Features, sess: &ParseSess,
134134
/// - `parsing_patterns`: same as [parse].
135135
/// - `sess`: the parsing session. Any errors will be emitted to this session.
136136
/// - `features`: language features so we can do feature gating.
137-
fn parse_tree(
138-
tree: tokenstream::TokenTree,
139-
outer_trees: &mut impl Iterator<Item = tokenstream::TokenTree>,
137+
fn parse_tree<'a>(
138+
tree: &'a tokenstream::TokenTree,
139+
outer_trees: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
140140
parsing_patterns: bool,
141141
sess: &ParseSess,
142142
node_id: NodeId,
@@ -146,21 +146,21 @@ fn parse_tree(
146146
// Depending on what `tree` is, we could be parsing different parts of a macro
147147
match tree {
148148
// `tree` is a `$` token. Look at the next token in `trees`
149-
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _) => {
149+
&tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _) => {
150150
// FIXME: Handle `Invisible`-delimited groups in a more systematic way
151151
// during parsing.
152152
let mut next = outer_trees.next();
153-
let mut trees: Box<dyn Iterator<Item = tokenstream::TokenTree>>;
153+
let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
154154
if let Some(tokenstream::TokenTree::Delimited(_, Delimiter::Invisible, tts)) = next {
155-
trees = Box::new(tts.into_trees());
155+
trees = Box::new(tts.trees());
156156
next = trees.next();
157157
} else {
158158
trees = Box::new(outer_trees);
159159
}
160160

161161
match next {
162162
// `tree` is followed by a delimited set of token trees.
163-
Some(tokenstream::TokenTree::Delimited(delim_span, delim, tts)) => {
163+
Some(&tokenstream::TokenTree::Delimited(delim_span, delim, ref tts)) => {
164164
if parsing_patterns {
165165
if delim != Delimiter::Parenthesis {
166166
span_dollar_dollar_or_metavar_in_the_lhs_err(
@@ -228,7 +228,7 @@ fn parse_tree(
228228
}
229229

230230
// `tree` is followed by another `$`. This is an escaped `$`.
231-
Some(tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _)) => {
231+
Some(&tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _)) => {
232232
if parsing_patterns {
233233
span_dollar_dollar_or_metavar_in_the_lhs_err(
234234
sess,
@@ -256,11 +256,11 @@ fn parse_tree(
256256
}
257257

258258
// `tree` is an arbitrary token. Keep it.
259-
tokenstream::TokenTree::Token(token, _) => TokenTree::Token(token),
259+
tokenstream::TokenTree::Token(token, _) => TokenTree::Token(token.clone()),
260260

261261
// `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
262262
// descend into the delimited set and further parse it.
263-
tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
263+
&tokenstream::TokenTree::Delimited(span, delim, ref tts) => TokenTree::Delimited(
264264
span,
265265
Delimited {
266266
delim,
@@ -286,16 +286,16 @@ fn kleene_op(token: &Token) -> Option<KleeneOp> {
286286
/// - Ok(Ok((op, span))) if the next token tree is a KleeneOp
287287
/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp
288288
/// - Err(span) if the next token tree is not a token
289-
fn parse_kleene_op(
290-
input: &mut impl Iterator<Item = tokenstream::TokenTree>,
289+
fn parse_kleene_op<'a>(
290+
input: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
291291
span: Span,
292292
) -> Result<Result<(KleeneOp, Span), Token>, Span> {
293293
match input.next() {
294294
Some(tokenstream::TokenTree::Token(token, _)) => match kleene_op(&token) {
295295
Some(op) => Ok(Ok((op, token.span))),
296-
None => Ok(Err(token)),
296+
None => Ok(Err(token.clone())),
297297
},
298-
tree => Err(tree.as_ref().map_or(span, tokenstream::TokenTree::span)),
298+
tree => Err(tree.map_or(span, tokenstream::TokenTree::span)),
299299
}
300300
}
301301

@@ -311,8 +311,8 @@ fn parse_kleene_op(
311311
/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene
312312
/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an
313313
/// error with the appropriate span is emitted to `sess` and a dummy value is returned.
314-
fn parse_sep_and_kleene_op(
315-
input: &mut impl Iterator<Item = tokenstream::TokenTree>,
314+
fn parse_sep_and_kleene_op<'a>(
315+
input: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
316316
span: Span,
317317
sess: &ParseSess,
318318
) -> (Option<Token>, KleeneToken) {

‎compiler/rustc_expand/src/parse/tests.rs

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,8 @@ fn bad_path_expr_1() {
6969
#[test]
7070
fn string_to_tts_macro() {
7171
create_default_session_globals_then(|| {
72-
let tts: Vec<_> =
73-
string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).into_trees().collect();
74-
let tts: &[TokenTree] = &tts[..];
72+
let stream = string_to_stream("macro_rules! zip (($a)=>($a))".to_string());
73+
let tts = &stream.trees().collect::<Vec<_>>()[..];
7574

7675
match tts {
7776
[
@@ -300,9 +299,7 @@ fn ttdelim_span() {
300299
.unwrap();
301300

302301
let ast::ExprKind::MacCall(mac) = &expr.kind else { panic!("not a macro") };
303-
let tts: Vec<_> = mac.args.tokens.clone().into_trees().collect();
304-
305-
let span = tts.iter().rev().next().unwrap().span();
302+
let span = mac.args.tokens.trees().last().unwrap().span();
306303

307304
match sess.source_map().span_to_snippet(span) {
308305
Ok(s) => assert_eq!(&s[..], "{ body }"),

‎compiler/rustc_expand/src/proc_macro_server.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,10 +94,10 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
9494
// Estimate the capacity as `stream.len()` rounded up to the next power
9595
// of two to limit the number of required reallocations.
9696
let mut trees = Vec::with_capacity(stream.len().next_power_of_two());
97-
let mut cursor = stream.into_trees();
97+
let mut cursor = stream.trees();
9898

9999
while let Some(tree) = cursor.next() {
100-
let (Token { kind, span }, joint) = match tree {
100+
let (Token { kind, span }, joint) = match tree.clone() {
101101
tokenstream::TokenTree::Delimited(span, delim, tts) => {
102102
let delimiter = pm::Delimiter::from_internal(delim);
103103
trees.push(TokenTree::Group(Group {

0 commit comments

Comments
 (0)