Auto merge of #14795 - Veykril:hl-macros, r=Veykril

fix: Add macro modifier for highlighting tokens in macro calls

Followup to #14777 we have to tell the client about the semantic tokens inside macro calls as those can be remapped. Adding a modifier will force this behavior.

Author: bors

crates/ide/src/syntax_highlighting.rs

@@ -166,6 +166,7 @@ pub struct HighlightConfig {
 // injected:: Emitted for doc-string injected highlighting like rust source blocks in documentation.
 // intraDocLink:: Emitted for intra doc links in doc-strings.
 // library:: Emitted for items that are defined outside of the current crate.
+// macro::  Emitted for tokens inside macro calls.
 // mutable:: Emitted for mutable locals and statics as well as functions taking `&mut self`.
 // public:: Emitted for items that are from the current crate and are `pub`.
 // reference:: Emitted for locals behind a reference and functions taking `self` by reference.
@@ -240,6 +241,7 @@ fn traverse(
     let mut current_macro: Option<ast::Macro> = None;
     let mut macro_highlighter = MacroHighlighter::default();
     let mut inside_attribute = false;
+    let mut inside_macro_call = false;
 
     // Walk all nodes, keeping track of whether we are inside a macro or not.
     // If in macro, expand it first and highlight the expanded code.
@@ -270,46 +272,50 @@ fn traverse(
                 inside_attribute = false
             }
 
-            Enter(NodeOrToken::Node(node)) if ast::Item::can_cast(node.kind()) => {
-                match ast::Item::cast(node.clone()) {
-                    Some(ast::Item::MacroRules(mac)) => {
-                        macro_highlighter.init();
-                        current_macro = Some(mac.into());
-                        continue;
-                    }
-                    Some(ast::Item::MacroDef(mac)) => {
-                        macro_highlighter.init();
-                        current_macro = Some(mac.into());
-                        continue;
-                    }
-                    Some(item) => {
-                        if matches!(node.kind(), FN | CONST | STATIC) {
-                            bindings_shadow_count.clear();
+            Enter(NodeOrToken::Node(node)) => match ast::Item::cast(node.clone()) {
+                Some(item) => {
+                    match item {
+                        ast::Item::MacroRules(mac) => {
+                            macro_highlighter.init();
+                            current_macro = Some(mac.into());
+                            continue;
+                        }
+                        ast::Item::MacroDef(mac) => {
+                            macro_highlighter.init();
+                            current_macro = Some(mac.into());
+                            continue;
                         }
+                        ast::Item::Fn(_) | ast::Item::Const(_) | ast::Item::Static(_) => {
+                            bindings_shadow_count.clear()
+                        }
+                        ast::Item::MacroCall(_) => {
+                            inside_macro_call = true;
+                        }
+                        _ => (),
+                    }
 
-                        if attr_or_derive_item.is_none() {
-                            if sema.is_attr_macro_call(&item) {
-                                attr_or_derive_item = Some(AttrOrDerive::Attr(item));
-                            } else {
-                                let adt = match item {
-                                    ast::Item::Enum(it) => Some(ast::Adt::Enum(it)),
-                                    ast::Item::Struct(it) => Some(ast::Adt::Struct(it)),
-                                    ast::Item::Union(it) => Some(ast::Adt::Union(it)),
-                                    _ => None,
-                                };
-                                match adt {
-                                    Some(adt) if sema.is_derive_annotated(&adt) => {
-                                        attr_or_derive_item =
-                                            Some(AttrOrDerive::Derive(ast::Item::from(adt)));
-                                    }
-                                    _ => (),
+                    if attr_or_derive_item.is_none() {
+                        if sema.is_attr_macro_call(&item) {
+                            attr_or_derive_item = Some(AttrOrDerive::Attr(item));
+                        } else {
+                            let adt = match item {
+                                ast::Item::Enum(it) => Some(ast::Adt::Enum(it)),
+                                ast::Item::Struct(it) => Some(ast::Adt::Struct(it)),
+                                ast::Item::Union(it) => Some(ast::Adt::Union(it)),
+                                _ => None,
+                            };
+                            match adt {
+                                Some(adt) if sema.is_derive_annotated(&adt) => {
+                                    attr_or_derive_item =
+                                        Some(AttrOrDerive::Derive(ast::Item::from(adt)));
                                 }
+                                _ => (),
                             }
                         }
                     }
-                    _ => (),
                 }
-            }
+                _ => (),
+            },
             Leave(NodeOrToken::Node(node)) if ast::Item::can_cast(node.kind()) => {
                 match ast::Item::cast(node.clone()) {
                     Some(ast::Item::MacroRules(mac)) => {
@@ -327,6 +333,9 @@ fn traverse(
                     {
                         attr_or_derive_item = None;
                     }
+                    Some(ast::Item::MacroCall(_)) => {
+                        inside_macro_call = false;
+                    }
                     _ => (),
                 }
             }
@@ -476,6 +485,9 @@ fn traverse(
             if inside_attribute {
                 highlight |= HlMod::Attribute
             }
+            if inside_macro_call && tt_level > 0 {
+                highlight |= HlMod::Macro
+            }
 
             hl.add(HlRange { range, highlight, binding_hash });
         }

crates/ide/src/syntax_highlighting/inject.rs

@@ -52,7 +52,11 @@ pub(super) fn ra_fixture(
 
         if let Some(next) = text.strip_prefix(marker) {
             if let Some(range) = literal.map_range_up(TextRange::at(offset, TextSize::of(marker))) {
-                hl.add(HlRange { range, highlight: HlTag::Keyword.into(), binding_hash: None });
+                hl.add(HlRange {
+                    range,
+                    highlight: HlTag::Keyword | HlMod::Injected,
+                    binding_hash: None,
+                });
             }
 
             text = next;
@@ -66,14 +70,24 @@ pub(super) fn ra_fixture(
 
     for mut hl_range in analysis
         .highlight(
-            HighlightConfig { syntactic_name_ref_highlighting: false, ..config },
+            HighlightConfig {
+                syntactic_name_ref_highlighting: false,
+                punctuation: true,
+                operator: true,
+                strings: true,
+                specialize_punctuation: config.specialize_punctuation,
+                specialize_operator: config.operator,
+                inject_doc_comment: config.inject_doc_comment,
+                macro_bang: config.macro_bang,
+            },
             tmp_file_id,
         )
         .unwrap()
     {
         for range in inj.map_range_up(hl_range.range) {
             if let Some(range) = literal.map_range_up(range) {
                 hl_range.range = range;
+                hl_range.highlight |= HlMod::Injected;
                 hl.add(hl_range);
             }
         }
@@ -217,7 +231,16 @@ pub(super) fn doc_comment(
     if let Ok(ranges) = analysis.with_db(|db| {
         super::highlight(
             db,
-            HighlightConfig { syntactic_name_ref_highlighting: true, ..config },
+            HighlightConfig {
+                syntactic_name_ref_highlighting: true,
+                punctuation: true,
+                operator: true,
+                strings: true,
+                specialize_punctuation: config.specialize_punctuation,
+                specialize_operator: config.operator,
+                inject_doc_comment: config.inject_doc_comment,
+                macro_bang: config.macro_bang,
+            },
             tmp_file_id,
             None,
         )

crates/ide/src/syntax_highlighting/tags.rs

@@ -49,7 +49,7 @@ pub enum HlMod {
     Associated = 0,
     /// Used with keywords like `async` and `await`.
     Async,
-    /// Used to differentiate individual elements within attributes.
+    /// Used to differentiate individual elements within attribute calls.
     Attribute,
     /// Callable item or value.
     Callable,
@@ -72,6 +72,8 @@ pub enum HlMod {
     IntraDocLink,
     /// Used for items from other crates.
     Library,
+    /// Used to differentiate individual elements within macro calls.
+    Macro,
     /// Mutable binding.
     Mutable,
     /// Used for public items.
@@ -200,7 +202,7 @@ impl fmt::Display for HlTag {
 }
 
 impl HlMod {
-    const ALL: &'static [HlMod; 19] = &[
+    const ALL: &'static [HlMod; HlMod::Unsafe as usize + 1] = &[
         HlMod::Associated,
         HlMod::Async,
         HlMod::Attribute,
@@ -214,6 +216,7 @@ impl HlMod {
         HlMod::Injected,
         HlMod::IntraDocLink,
         HlMod::Library,
+        HlMod::Macro,
         HlMod::Mutable,
         HlMod::Public,
         HlMod::Reference,
@@ -237,6 +240,7 @@ impl HlMod {
             HlMod::Injected => "injected",
             HlMod::IntraDocLink => "intra_doc_link",
             HlMod::Library => "library",
+            HlMod::Macro => "macro",
             HlMod::Mutable => "mutable",
             HlMod::Public => "public",
             HlMod::Reference => "reference",

crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html

@@ -93,7 +93,7 @@
     <span class="comment documentation">///</span><span class="comment documentation"> </span><span class="keyword injected">let</span><span class="none injected"> </span><span class="variable declaration injected">foo</span><span class="none injected"> </span><span class="operator injected">=</span><span class="none injected"> </span><span class="struct injected">Foo</span><span class="operator injected">::</span><span class="function injected">new</span><span class="parenthesis injected">(</span><span class="parenthesis injected">)</span><span class="semicolon injected">;</span>
     <span class="comment documentation">///</span>
     <span class="comment documentation">///</span><span class="comment documentation"> </span><span class="comment injected">// calls bar on foo</span>
-    <span class="comment documentation">///</span><span class="comment documentation"> </span><span class="macro injected">assert</span><span class="macro_bang injected">!</span><span class="parenthesis injected">(</span><span class="none injected">foo</span><span class="operator injected">.</span><span class="none injected">bar</span><span class="parenthesis injected">(</span><span class="parenthesis injected">)</span><span class="parenthesis injected">)</span><span class="semicolon injected">;</span>
+    <span class="comment documentation">///</span><span class="comment documentation"> </span><span class="macro injected">assert</span><span class="macro_bang injected">!</span><span class="parenthesis injected macro">(</span><span class="none injected macro">foo</span><span class="operator injected macro">.</span><span class="none injected macro">bar</span><span class="parenthesis injected macro">(</span><span class="parenthesis injected macro">)</span><span class="parenthesis injected macro">)</span><span class="semicolon injected">;</span>
     <span class="comment documentation">///</span>
     <span class="comment documentation">///</span><span class="comment documentation"> </span><span class="keyword injected">let</span><span class="none injected"> </span><span class="variable declaration injected">bar</span><span class="none injected"> </span><span class="operator injected">=</span><span class="none injected"> </span><span class="variable injected">foo</span><span class="operator injected">.</span><span class="field injected">bar</span><span class="none injected"> </span><span class="logical injected">||</span><span class="none injected"> </span><span class="struct injected">Foo</span><span class="operator injected">::</span><span class="constant injected">bar</span><span class="semicolon injected">;</span>
     <span class="comment documentation">///</span>
@@ -145,7 +145,7 @@
 
 <span class="comment documentation">/// ```</span>
 <span class="comment documentation">///</span><span class="comment documentation"> </span><span class="keyword injected">macro_rules</span><span class="macro_bang injected">!</span><span class="none injected"> </span><span class="macro declaration injected">noop</span><span class="none injected"> </span><span class="brace injected">{</span><span class="none injected"> </span><span class="parenthesis injected">(</span><span class="punctuation injected">$</span><span class="none injected">expr</span><span class="colon injected">:</span><span class="none injected">expr</span><span class="parenthesis injected">)</span><span class="none injected"> </span><span class="operator injected">=</span><span class="angle injected">&gt;</span><span class="none injected"> </span><span class="brace injected">{</span><span class="none injected"> </span><span class="punctuation injected">$</span><span class="none injected">expr </span><span class="brace injected">}</span><span class="brace injected">}</span>
-<span class="comment documentation">///</span><span class="comment documentation"> </span><span class="macro injected">noop</span><span class="macro_bang injected">!</span><span class="parenthesis injected">(</span><span class="numeric_literal injected">1</span><span class="parenthesis injected">)</span><span class="semicolon injected">;</span>
+<span class="comment documentation">///</span><span class="comment documentation"> </span><span class="macro injected">noop</span><span class="macro_bang injected">!</span><span class="parenthesis injected macro">(</span><span class="numeric_literal injected macro">1</span><span class="parenthesis injected macro">)</span><span class="semicolon injected">;</span>
 <span class="comment documentation">/// ```</span>
 <span class="keyword">macro_rules</span><span class="macro_bang">!</span> <span class="macro declaration">noop</span> <span class="brace">{</span>
     <span class="parenthesis">(</span><span class="punctuation">$</span>expr<span class="colon">:</span>expr<span class="parenthesis">)</span> <span class="operator">=</span><span class="angle">&gt;</span> <span class="brace">{</span>
@@ -165,7 +165,7 @@
 <span class="comment documentation">///</span>
 <span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="builtin_attr attribute library">cfg_attr</span><span class="parenthesis attribute">(</span><span class="none attribute">feature</span> <span class="operator attribute">=</span> <span class="string_literal attribute">"alloc"</span><span class="comma attribute">,</span> <span class="none attribute">doc</span> <span class="operator attribute">=</span> <span class="string_literal attribute">"```rust"</span><span class="parenthesis attribute">)</span><span class="attribute_bracket attribute">]</span>
 <span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="builtin_attr attribute library">cfg_attr</span><span class="parenthesis attribute">(</span><span class="none attribute">not</span><span class="parenthesis attribute">(</span><span class="none attribute">feature</span> <span class="operator attribute">=</span> <span class="string_literal attribute">"alloc"</span><span class="parenthesis attribute">)</span><span class="comma attribute">,</span> <span class="none attribute">doc</span> <span class="operator attribute">=</span> <span class="string_literal attribute">"```ignore"</span><span class="parenthesis attribute">)</span><span class="attribute_bracket attribute">]</span>
-<span class="comment documentation">///</span><span class="comment documentation"> </span><span class="keyword injected">let</span><span class="none injected"> </span><span class="punctuation injected">_</span><span class="none injected"> </span><span class="operator injected">=</span><span class="none injected"> </span><span class="function injected">example</span><span class="parenthesis injected">(</span><span class="operator injected">&</span><span class="module injected">alloc</span><span class="operator injected">::</span><span class="macro injected">vec</span><span class="macro_bang injected">!</span><span class="bracket injected">[</span><span class="numeric_literal injected">1</span><span class="comma injected">,</span><span class="none injected"> </span><span class="numeric_literal injected">2</span><span class="comma injected">,</span><span class="none injected"> </span><span class="numeric_literal injected">3</span><span class="bracket injected">]</span><span class="parenthesis injected">)</span><span class="semicolon injected">;</span>
+<span class="comment documentation">///</span><span class="comment documentation"> </span><span class="keyword injected">let</span><span class="none injected"> </span><span class="punctuation injected">_</span><span class="none injected"> </span><span class="operator injected">=</span><span class="none injected"> </span><span class="function injected">example</span><span class="parenthesis injected">(</span><span class="operator injected">&</span><span class="module injected">alloc</span><span class="operator injected">::</span><span class="macro injected">vec</span><span class="macro_bang injected">!</span><span class="bracket injected macro">[</span><span class="numeric_literal injected macro">1</span><span class="comma injected macro">,</span><span class="none injected"> </span><span class="numeric_literal injected macro">2</span><span class="comma injected macro">,</span><span class="none injected"> </span><span class="numeric_literal injected macro">3</span><span class="bracket injected macro">]</span><span class="parenthesis injected">)</span><span class="semicolon injected">;</span>
 <span class="comment documentation">/// ```</span>
 <span class="keyword">pub</span> <span class="keyword">fn</span> <span class="function declaration public">mix_and_match</span><span class="parenthesis">(</span><span class="parenthesis">)</span> <span class="brace">{</span><span class="brace">}</span>