rust_analyzer/lsp/
semantic_tokens.rs

1//! Semantic Tokens helpers
2
3use std::ops;
4
5use lsp_types::{
6    Range, SemanticToken, SemanticTokenModifier, SemanticTokenType, SemanticTokens,
7    SemanticTokensEdit,
8};
9
10macro_rules! define_semantic_token_types {
11    (
12        standard {
13            $($standard:ident),*$(,)?
14        }
15        custom {
16            $(($custom:ident, $string:literal) $(=> $fallback:ident)?),*$(,)?
17        }
18
19    ) => {
20        pub(crate) mod types {
21            use super::SemanticTokenType;
22            $(pub(crate) const $standard: SemanticTokenType = SemanticTokenType::$standard;)*
23            $(pub(crate) const $custom: SemanticTokenType = SemanticTokenType::new($string);)*
24        }
25
26        pub(crate) const SUPPORTED_TYPES: &[SemanticTokenType] = &[
27            $(self::types::$standard,)*
28            $(self::types::$custom),*
29        ];
30
31        pub(crate) fn standard_fallback_type(token: SemanticTokenType) -> Option<SemanticTokenType> {
32            use self::types::*;
33            $(
34                if token == $custom {
35                    None $(.or(Some(self::types::$fallback)))?
36                } else
37            )*
38            { Some(token )}
39        }
40    };
41}
42
43define_semantic_token_types![
44    standard {
45        COMMENT,
46        DECORATOR,
47        ENUM_MEMBER,
48        ENUM,
49        FUNCTION,
50        INTERFACE,
51        KEYWORD,
52        MACRO,
53        METHOD,
54        NAMESPACE,
55        NUMBER,
56        OPERATOR,
57        PARAMETER,
58        PROPERTY,
59        STRING,
60        STRUCT,
61        TYPE_PARAMETER,
62        VARIABLE,
63        TYPE,
64    }
65
66    custom {
67        (ANGLE, "angle"),
68        (ARITHMETIC, "arithmetic") => OPERATOR,
69        (ATTRIBUTE_BRACKET, "attributeBracket") => DECORATOR,
70        (ATTRIBUTE, "attribute") => DECORATOR,
71        (BITWISE, "bitwise") => OPERATOR,
72        (BOOLEAN, "boolean"),
73        (BRACE, "brace"),
74        (BRACKET, "bracket"),
75        (BUILTIN_ATTRIBUTE, "builtinAttribute") => DECORATOR,
76        (BUILTIN_TYPE, "builtinType") => TYPE,
77        (CHAR, "character") => STRING,
78        (COLON, "colon"),
79        (COMMA, "comma"),
80        (COMPARISON, "comparison") => OPERATOR,
81        (CONST_PARAMETER, "constParameter"),
82        (CONST, "const") => VARIABLE,
83        (DERIVE_HELPER, "deriveHelper") => DECORATOR,
84        (DERIVE, "derive") => DECORATOR,
85        (DOT, "dot"),
86        (ESCAPE_SEQUENCE, "escapeSequence") => STRING,
87        (FORMAT_SPECIFIER, "formatSpecifier") => STRING,
88        (GENERIC, "generic") => TYPE_PARAMETER,
89        (INVALID_ESCAPE_SEQUENCE, "invalidEscapeSequence") => STRING,
90        (LABEL, "label"),
91        (LIFETIME, "lifetime"),
92        (LOGICAL, "logical") => OPERATOR,
93        (MACRO_BANG, "macroBang") => MACRO,
94        (NEGATION, "negation") => OPERATOR,
95        (PARENTHESIS, "parenthesis"),
96        (PROC_MACRO, "procMacro") => MACRO,
97        (PUNCTUATION, "punctuation"),
98        (SELF_KEYWORD, "selfKeyword") => KEYWORD,
99        (SELF_TYPE_KEYWORD, "selfTypeKeyword") => KEYWORD,
100        (SEMICOLON, "semicolon"),
101        (STATIC, "static") => VARIABLE,
102        (TOOL_MODULE, "toolModule") => DECORATOR,
103        (TYPE_ALIAS, "typeAlias") => TYPE,
104        (UNION, "union") => TYPE,
105        (UNRESOLVED_REFERENCE, "unresolvedReference"),
106    }
107];
108
109macro_rules! count_tts {
110    () => {0usize};
111    ($_head:tt $($tail:tt)*) => {1usize + count_tts!($($tail)*)};
112}
113macro_rules! define_semantic_token_modifiers {
114    (
115        standard {
116            $($standard:ident),*$(,)?
117        }
118        custom {
119            $(($custom:ident, $string:literal)),*$(,)?
120        }
121
122    ) => {
123        pub(crate) mod modifiers {
124            use super::SemanticTokenModifier;
125
126            $(pub(crate) const $standard: SemanticTokenModifier = SemanticTokenModifier::$standard;)*
127            $(pub(crate) const $custom: SemanticTokenModifier = SemanticTokenModifier::new($string);)*
128        }
129
130        pub(crate) const SUPPORTED_MODIFIERS: &[SemanticTokenModifier] = &[
131            $(SemanticTokenModifier::$standard,)*
132            $(self::modifiers::$custom),*
133        ];
134
135        const LAST_STANDARD_MOD: usize = count_tts!($($standard)*);
136    };
137}
138
139define_semantic_token_modifiers![
140    standard {
141        ASYNC,
142        DOCUMENTATION,
143        DECLARATION,
144        STATIC,
145        DEFAULT_LIBRARY,
146        DEPRECATED,
147    }
148    custom {
149        (ASSOCIATED, "associated"),
150        (ATTRIBUTE_MODIFIER, "attribute"),
151        (CALLABLE, "callable"),
152        (CONSTANT, "constant"),
153        (CONSUMING, "consuming"),
154        (CONTROL_FLOW, "controlFlow"),
155        (CRATE_ROOT, "crateRoot"),
156        (INJECTED, "injected"),
157        (INTRA_DOC_LINK, "intraDocLink"),
158        (LIBRARY, "library"),
159        (MACRO_MODIFIER, "macro"),
160        (MUTABLE, "mutable"),
161        (PROC_MACRO_MODIFIER, "procMacro"),
162        (PUBLIC, "public"),
163        (REFERENCE, "reference"),
164        (TRAIT_MODIFIER, "trait"),
165        (UNSAFE, "unsafe"),
166    }
167];
168
169#[derive(Default)]
170pub(crate) struct ModifierSet(pub(crate) u32);
171
172impl ModifierSet {
173    pub(crate) fn standard_fallback(&mut self) {
174        // Remove all non standard modifiers
175        self.0 &= !(!0u32 << LAST_STANDARD_MOD)
176    }
177}
178
179impl ops::BitOrAssign<SemanticTokenModifier> for ModifierSet {
180    fn bitor_assign(&mut self, rhs: SemanticTokenModifier) {
181        let idx = SUPPORTED_MODIFIERS.iter().position(|it| it == &rhs).unwrap();
182        self.0 |= 1 << idx;
183    }
184}
185
186/// Tokens are encoded relative to each other.
187///
188/// This is a direct port of <https://github.com/microsoft/vscode-languageserver-node/blob/f425af9de46a0187adb78ec8a46b9b2ce80c5412/server/src/sematicTokens.proposed.ts#L45>
189pub(crate) struct SemanticTokensBuilder {
190    id: String,
191    prev_line: u32,
192    prev_char: u32,
193    data: Vec<SemanticToken>,
194}
195
196impl SemanticTokensBuilder {
197    pub(crate) fn new(id: String) -> Self {
198        SemanticTokensBuilder { id, prev_line: 0, prev_char: 0, data: Default::default() }
199    }
200
201    /// Push a new token onto the builder
202    pub(crate) fn push(&mut self, range: Range, token_index: u32, modifier_bitset: u32) {
203        let mut push_line = range.start.line;
204        let mut push_char = range.start.character;
205
206        if !self.data.is_empty() {
207            push_line -= self.prev_line;
208            if push_line == 0 {
209                push_char -= self.prev_char;
210            }
211        }
212
213        // A token cannot be multiline
214        let token_len = range.end.character - range.start.character;
215
216        let token = SemanticToken {
217            delta_line: push_line,
218            delta_start: push_char,
219            length: token_len,
220            token_type: token_index,
221            token_modifiers_bitset: modifier_bitset,
222        };
223
224        self.data.push(token);
225
226        self.prev_line = range.start.line;
227        self.prev_char = range.start.character;
228    }
229
230    pub(crate) fn build(self) -> SemanticTokens {
231        SemanticTokens { result_id: Some(self.id), data: self.data }
232    }
233}
234
235pub(crate) fn diff_tokens(old: &[SemanticToken], new: &[SemanticToken]) -> Vec<SemanticTokensEdit> {
236    let offset = new.iter().zip(old.iter()).take_while(|&(n, p)| n == p).count();
237
238    let (_, old) = old.split_at(offset);
239    let (_, new) = new.split_at(offset);
240
241    let offset_from_end =
242        new.iter().rev().zip(old.iter().rev()).take_while(|&(n, p)| n == p).count();
243
244    let (old, _) = old.split_at(old.len() - offset_from_end);
245    let (new, _) = new.split_at(new.len() - offset_from_end);
246
247    if old.is_empty() && new.is_empty() {
248        vec![]
249    } else {
250        // The lsp data field is actually a byte-diff but we
251        // travel in tokens so `start` and `delete_count` are in multiples of the
252        // serialized size of `SemanticToken`.
253        vec![SemanticTokensEdit {
254            start: 5 * offset as u32,
255            delete_count: 5 * old.len() as u32,
256            data: Some(new.into()),
257        }]
258    }
259}
260
261pub(crate) fn type_index(ty: SemanticTokenType) -> u32 {
262    SUPPORTED_TYPES.iter().position(|it| *it == ty).unwrap() as u32
263}
264
265#[cfg(test)]
266mod tests {
267    use super::*;
268
269    fn from(t: (u32, u32, u32, u32, u32)) -> SemanticToken {
270        SemanticToken {
271            delta_line: t.0,
272            delta_start: t.1,
273            length: t.2,
274            token_type: t.3,
275            token_modifiers_bitset: t.4,
276        }
277    }
278
279    #[test]
280    fn test_diff_insert_at_end() {
281        let before = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
282        let after = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10)), from((11, 12, 13, 14, 15))];
283
284        let edits = diff_tokens(&before, &after);
285        assert_eq!(
286            edits[0],
287            SemanticTokensEdit {
288                start: 10,
289                delete_count: 0,
290                data: Some(vec![from((11, 12, 13, 14, 15))])
291            }
292        );
293    }
294
295    #[test]
296    fn test_diff_insert_at_beginning() {
297        let before = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
298        let after = [from((11, 12, 13, 14, 15)), from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
299
300        let edits = diff_tokens(&before, &after);
301        assert_eq!(
302            edits[0],
303            SemanticTokensEdit {
304                start: 0,
305                delete_count: 0,
306                data: Some(vec![from((11, 12, 13, 14, 15))])
307            }
308        );
309    }
310
311    #[test]
312    fn test_diff_insert_in_middle() {
313        let before = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
314        let after = [
315            from((1, 2, 3, 4, 5)),
316            from((10, 20, 30, 40, 50)),
317            from((60, 70, 80, 90, 100)),
318            from((6, 7, 8, 9, 10)),
319        ];
320
321        let edits = diff_tokens(&before, &after);
322        assert_eq!(
323            edits[0],
324            SemanticTokensEdit {
325                start: 5,
326                delete_count: 0,
327                data: Some(vec![from((10, 20, 30, 40, 50)), from((60, 70, 80, 90, 100))])
328            }
329        );
330    }
331
332    #[test]
333    fn test_diff_remove_from_end() {
334        let before = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10)), from((11, 12, 13, 14, 15))];
335        let after = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
336
337        let edits = diff_tokens(&before, &after);
338        assert_eq!(edits[0], SemanticTokensEdit { start: 10, delete_count: 5, data: Some(vec![]) });
339    }
340
341    #[test]
342    fn test_diff_remove_from_beginning() {
343        let before = [from((11, 12, 13, 14, 15)), from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
344        let after = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
345
346        let edits = diff_tokens(&before, &after);
347        assert_eq!(edits[0], SemanticTokensEdit { start: 0, delete_count: 5, data: Some(vec![]) });
348    }
349
350    #[test]
351    fn test_diff_remove_from_middle() {
352        let before = [
353            from((1, 2, 3, 4, 5)),
354            from((10, 20, 30, 40, 50)),
355            from((60, 70, 80, 90, 100)),
356            from((6, 7, 8, 9, 10)),
357        ];
358        let after = [from((1, 2, 3, 4, 5)), from((6, 7, 8, 9, 10))];
359
360        let edits = diff_tokens(&before, &after);
361        assert_eq!(edits[0], SemanticTokensEdit { start: 5, delete_count: 10, data: Some(vec![]) });
362    }
363}
rust_analyzer/lsp/semantic_tokens.rs

rust_analyzer/lsp/
semantic_tokens.rs
