rust@ee53c6e (original) (raw)

`@@ -10,6 +10,7 @@ use std::collections::VecDeque;

10

`use std::fmt::{self, Display, Write};

11

`use std::iter;

12

13

use arrayvec::ArrayVec;

13

14

`use rustc_data_structures::fx::FxIndexMap;

14

15

`use rustc_lexer::{Cursor, FrontmatterAllowed, LiteralKind, TokenKind};

15

16

`use rustc_span::edition::Edition;

`@@ -818,7 +819,7 @@ impl<'src> Classifier<'src> {

818

819

820

821

`/// Concatenate colons and idents as one when possible.

821

fn get_full_ident_path(&mut self) -> Vec<(TokenKind, usize, usize)> {

822

fn get_full_ident_path(&mut self) -> ArrayVec<(TokenKind, usize, usize), 2> {

822

823

`let start = self.byte_pos as usize;

823

824

`let mut pos = start;

824

825

`let mut has_ident = false;

`@@ -832,13 +833,16 @@ impl<'src> Classifier<'src> {

832

833

`// Ident path can start with "::" but if we already have content in the ident path,

833

834

`// the "::" is mandatory.

834

835

`if has_ident && nb == 0 {

835

return vec![(TokenKind::Ident, start, pos)];

836

return ArrayVec::from_iter([(TokenKind::Ident, start, pos)]);

836

837

`} else if nb != 0 && nb != 2 {

837

if has_ident {

838

return vec![(TokenKind::Ident, start, pos), (TokenKind::Colon, pos, pos + nb)];

838

return if has_ident {

839

ArrayVec::from_iter([

840

(TokenKind::Ident, start, pos),

841

(TokenKind::Colon, pos, pos + nb),

842

])

839

843

`} else {

840

return vec![(TokenKind::Colon, start, pos + nb)];

841

}

844

ArrayVec::from_iter([(TokenKind::Colon, start, pos + nb)])

845

};

842

846

843

847

844

848

`if let Some((None, text)) = self.tokens.peek().map(|(token, text)| {

`@@ -854,15 +858,21 @@ impl<'src> Classifier<'src> {

854

858

` pos += text.len() + nb;

855

859

` has_ident = true;

856

860

`self.tokens.next();

857

} else if nb > 0 && has_ident {

858

return vec![(TokenKind::Ident, start, pos), (TokenKind::Colon, pos, pos + nb)];

861

continue;

862

}

863

+

864

return if nb > 0 && has_ident {

865

ArrayVec::from_iter([

866

(TokenKind::Ident, start, pos),

867

(TokenKind::Colon, pos, pos + nb),

868

])

859

869

`} else if nb > 0 {

860

return vec![(TokenKind::Colon, start, start + nb)];

870

ArrayVec::from_iter([(TokenKind::Colon, start, start + nb)])

861

871

`} else if has_ident {

862

return vec![(TokenKind::Ident, start, pos)];

872

ArrayVec::from_iter([(TokenKind::Ident, start, pos)])

863

873

`} else {

864

return Vec::new();

865

}

874

ArrayVec::new()

875

};

866

876

867

877

868

878

`@@ -885,7 +895,7 @@ impl<'src> Classifier<'src> {

885

895

`/// The general structure for this method is to iterate over each token,

886

896

`/// possibly giving it an HTML span with a class specifying what flavor of

887

897

`/// token is used.

888

fn highlight(mut self, sink: &mut dyn FnMut(Span, Highlight<'src>)) {

898

fn highlight(mut self, mut sink: impl FnMut(Span, Highlight<'src>)) {

889

899

`loop {

890

900

`if let Some(decs) = self.decorations.as_mut() {

891

901

`let byte_pos = self.byte_pos;

`@@ -903,21 +913,20 @@ impl<'src> Classifier<'src> {

903

913

`if self

904

914

`.tokens

905

915

`.peek()

906

.map(|t| matches!(t.0, TokenKind::Colon | TokenKind::Ident))

907

.unwrap_or(false)

916

.is_some_and(|(kind, _)| matches!(kind, TokenKind::Colon | TokenKind::Ident))

908

917

909

918

`let tokens = self.get_full_ident_path();

910

919

`for (token, start, end) in &tokens {

911

920

`let text = &self.src[*start..*end];

912

self.advance(*token, text, sink, *start as u32);

921

self.advance(*token, text, &mut sink, *start as u32);

913

922

`self.byte_pos += text.len() as u32;

914

923

915

924

`if !tokens.is_empty() {

916

925

`continue;

917

926

918

927

919

928

`if let Some((token, text, before)) = self.next() {

920

self.advance(token, text, sink, before);

929

self.advance(token, text, &mut sink, before);

921

930

`} else {

922

931

`break;

923

932

`@@ -934,26 +943,28 @@ impl<'src> Classifier<'src> {

934

943

`&mut self,

935

944

`token: TokenKind,

936

945

`text: &'src str,

937

sink: &mut dyn FnMut(Span, Highlight<'src>),

946

mut sink: impl FnMut(Span, Highlight<'src>),

938

947

`before: u32,

939

948

`) {

940

949

`let lookahead = self.peek();

941

950

`let file_span = self.file_span;

942

let no_highlight = |sink: &mut dyn FnMut(_, _)| {

943

sink(new_span(before, text, file_span), Highlight::Token { text, class: None })

944

};

945

let whitespace = |sink: &mut dyn FnMut(_, _)| {

951

let no_highlight =

952

|| (new_span(before, text, file_span), Highlight::Token { text, class: None });

953

let mut whitespace = |class| {

946

954

`let mut start = 0u32;

947

955

`for part in text.split('\n').intersperse("\n").filter(|s| !s.is_empty()) {

948

956

`sink(

949

957

`new_span(before + start, part, file_span),

950

Highlight::Token { text: part, class: None },

958

Highlight::Token { text: part, class },

951

959

`);

952

960

` start += part.len() as u32;

953

961

954

962

`};

955

963

`let class = match token {

956

TokenKind::Whitespace => return whitespace(sink),

964

TokenKind::Whitespace => {

965

whitespace(None);

966

return;

967

}

957

968

`TokenKind::LineComment { doc_style } | TokenKind::BlockComment { doc_style, .. } => {

958

969

`if doc_style.is_some() {

959

970

`Class::DocComment

`@@ -974,7 +985,10 @@ impl<'src> Classifier<'src> {

974

985

`// or a reference or pointer type. Unless, of course, it looks like

975

986

`` // a logical and or a multiplication operator: && or * .

976

987

`TokenKind::Star => match self.tokens.peek() {

977

Some((TokenKind::Whitespace, _)) => return whitespace(sink),

988

Some((TokenKind::Whitespace, _)) => {

989

whitespace(None);

990

return;

991

}

978

992

`Some((TokenKind::Ident, "mut")) => {

979

993

`self.next();

980

994

`sink(

`@@ -1004,7 +1018,10 @@ impl<'src> Classifier<'src> {

1004

1018

`sink(DUMMY_SP, Highlight::Token { text: "&=", class: None });

1005

1019

`return;

1006

1020

1007

Some((TokenKind::Whitespace, _)) => return whitespace(sink),

1021

Some((TokenKind::Whitespace, _)) => {

1022

whitespace(None);

1023

return;

1024

}

1008

1025

`Some((TokenKind::Ident, "mut")) => {

1009

1026

`self.next();

1010

1027

`sink(

`@@ -1028,7 +1045,11 @@ impl<'src> Classifier<'src> {

1028

1045

`sink(DUMMY_SP, Highlight::Token { text: "=>", class: None });

1029

1046

`return;

1030

1047

1031

_ => return no_highlight(sink),

1048

_ => {

1049

let (span, highlight) = no_highlight();

1050

sink(span, highlight);

1051

return;

1052

}

1032

1053

`},

1033

1054

`TokenKind::Minus if lookahead == Some(TokenKind::Gt) => {

1034

1055

`self.next();

`@@ -1045,7 +1066,11 @@ impl<'src> Classifier<'src> {

1045

1066

` | TokenKind::Percent

1046

1067

` | TokenKind::Bang

1047

1068

` | TokenKind::Lt

1048

| TokenKind::Gt => return no_highlight(sink),

1069

| TokenKind::Gt => {

1070

let (span, highlight) = no_highlight();

1071

sink(span, highlight);

1072

return;

1073

}

1049

1074

1050

1075

`// Miscellaneous, no highlighting.

1051

1076

`TokenKind::Dot

`@@ -1060,7 +1085,11 @@ impl<'src> Classifier<'src> {

1060

1085

` | TokenKind::Tilde

1061

1086

` | TokenKind::Colon

1062

1087

` | TokenKind::Frontmatter { .. }

1063

| TokenKind::Unknown => return no_highlight(sink),

1088

| TokenKind::Unknown => {

1089

let (span, highlight) = no_highlight();

1090

sink(span, highlight);

1091

return;

1092

}

1064

1093

1065

1094

`TokenKind::Question => Class::QuestionMark,

1066

1095

`@@ -1069,7 +1098,11 @@ impl<'src> Classifier<'src> {

1069

1098

`self.in_macro_nonterminal = true;

1070

1099

`Class::MacroNonTerminal

1071

1100

1072

_ => return no_highlight(sink),

1101

_ => {

1102

let (span, highlight) = no_highlight();

1103

sink(span, highlight);

1104

return;

1105

}

1073

1106

`},

1074

1107

1075

1108

`// This might be the start of an attribute. We're going to want to

`@@ -1100,9 +1133,11 @@ impl<'src> Classifier<'src> {

1100

1133

`Highlight::EnterSpan { class: Class::Attribute },

1101

1134

`);

1102

1135

1103

_ => (),

1136

_ => {}

1104

1137

1105

return no_highlight(sink);

1138

let (span, highlight) = no_highlight();

1139

sink(span, highlight);

1140

return;

1106

1141

1107

1142

`TokenKind::CloseBracket => {

1108

1143

`if self.in_attribute {

`@@ -1114,7 +1149,9 @@ impl<'src> Classifier<'src> {

1114

1149

`sink(DUMMY_SP, Highlight::ExitSpan);

1115

1150

`return;

1116

1151

1117

return no_highlight(sink);

1152

let (span, highlight) = no_highlight();

1153

sink(span, highlight);

1154

return;

1118

1155

1119

1156

`TokenKind::Literal { kind, .. } => match kind {

1120

1157

`// Text literals.

`@@ -1129,7 +1166,11 @@ impl<'src> Classifier<'src> {

1129

1166

`// Number literals.

1130

1167

`LiteralKind::Float { .. } | LiteralKind::Int { .. } => Class::Number,

1131

1168

`},

1132

TokenKind::GuardedStrPrefix => return no_highlight(sink),

1169

TokenKind::GuardedStrPrefix => {

1170

let (span, highlight) = no_highlight();

1171

sink(span, highlight);

1172

return;

1173

}

1133

1174

`TokenKind::Ident | TokenKind::RawIdent if lookahead == Some(TokenKind::Bang) => {

1134

1175

`self.in_macro = true;

1135

1176

`let span = new_span(before, text, file_span);

`@@ -1165,14 +1206,7 @@ impl<'src> Classifier<'src> {

1165

1206

`};

1166

1207

`// Anything that didn't return above is the simple case where we the

1167

1208

`` // class just spans a single token, so we can use the string method.

1168

let mut start = 0u32;

1169

for part in text.split('\n').intersperse("\n").filter(|s| !s.is_empty()) {

1170

sink(

1171

new_span(before + start, part, file_span),

1172

Highlight::Token { text: part, class: Some(class) },

1173

);

1174

start += part.len() as u32;

1175

}

1209

whitespace(Some(class));

1176

1210

1177

1211

1178

1212

`fn peek(&mut self) -> Option {