fe_parser/lexer/
token.rs

1use crate::node::Node;
2use crate::node::Span;
3use logos::Logos;
4use smol_str::SmolStr;
5use std::ops::Add;
6
7#[derive(Debug, PartialEq, Eq, Clone)]
8pub struct Token<'a> {
9    pub kind: TokenKind,
10    pub text: &'a str,
11    pub span: Span,
12}
13
14impl<'a> From<Token<'a>> for Node<SmolStr> {
15    fn from(tok: Token<'a>) -> Node<SmolStr> {
16        Node::new(tok.text.into(), tok.span)
17    }
18}
19
20impl<'a> Add<&Token<'a>> for Span {
21    type Output = Self;
22
23    fn add(self, other: &Token<'a>) -> Self {
24        self + other.span
25    }
26}
27
28#[derive(Debug, Copy, Clone, PartialEq, Eq, Logos)]
29pub enum TokenKind {
30    // Ignoring comments and spaces/tabs for now.
31    // If we implement an auto-formatting tool, we'll probably want to change this.
32    #[regex(r"//[^\n]*", logos::skip)]
33    #[regex("[ \t]+", logos::skip)]
34    #[error]
35    Error,
36
37    #[regex(r"\n[ \t]*")]
38    Newline,
39
40    #[regex("[a-zA-Z_][a-zA-Z0-9_]*")]
41    Name,
42    #[regex("[0-9]+(?:_[0-9]+)*")]
43    Int,
44    #[regex("0[xX][0-9a-fA-F]+")]
45    Hex,
46    #[regex("0[oO][0-7]+")]
47    Octal,
48    #[regex("0[bB][0-1]+")]
49    Binary,
50    // Float,
51    #[regex(r#""([^"\\]|\\.)*""#)]
52    #[regex(r#"'([^'\\]|\\.)*'"#)]
53    Text,
54    #[token("true")]
55    True,
56    #[token("false")]
57    False,
58    // #[token("None")] // ?
59    // None,
60    #[token("assert")]
61    Assert,
62    #[token("break")]
63    Break,
64    #[token("continue")]
65    Continue,
66    #[token("contract")]
67    Contract,
68    #[token("fn")]
69    Fn,
70    #[token("const")]
71    Const,
72    #[token("else")]
73    Else,
74    #[token("idx")]
75    Idx,
76    #[token("if")]
77    If,
78    #[token("match")]
79    Match,
80    #[token("impl")]
81    Impl,
82    #[token("pragma")]
83    Pragma,
84    #[token("for")]
85    For,
86    #[token("pub")]
87    Pub,
88    #[token("return")]
89    Return,
90    #[token("revert")]
91    Revert,
92    #[token("Self")]
93    SelfType,
94    #[token("self")]
95    SelfValue,
96    #[token("struct")]
97    Struct,
98    #[token("enum")]
99    Enum,
100    #[token("trait")]
101    Trait,
102    #[token("type")]
103    Type,
104    #[token("unsafe")]
105    Unsafe,
106    #[token("while")]
107    While,
108
109    #[token("and")]
110    And,
111    #[token("as")]
112    As,
113    #[token("in")]
114    In,
115    #[token("not")]
116    Not,
117    #[token("or")]
118    Or,
119    #[token("let")]
120    Let,
121    #[token("mut")]
122    Mut,
123    #[token("use")]
124    Use,
125    // Symbols
126    #[token("(")]
127    ParenOpen,
128    #[token(")")]
129    ParenClose,
130    #[token("[")]
131    BracketOpen,
132    #[token("]")]
133    BracketClose,
134    #[token("{")]
135    BraceOpen,
136    #[token("}")]
137    BraceClose,
138    #[token(":")]
139    Colon,
140    #[token("::")]
141    ColonColon,
142    #[token(",")]
143    Comma,
144    #[token("#")]
145    Hash,
146    #[token(";")]
147    Semi,
148    #[token("+")]
149    Plus,
150    #[token("-")]
151    Minus,
152    #[token("*")]
153    Star,
154    #[token("/")]
155    Slash,
156    #[token("|")]
157    Pipe,
158    #[token("&")]
159    Amper,
160    #[token("<")]
161    Lt,
162    #[token("<<")]
163    LtLt,
164    #[token(">")]
165    Gt,
166    #[token(">>")]
167    GtGt,
168    #[token("=")]
169    Eq,
170    #[token(".")]
171    Dot,
172    #[token("..")]
173    DotDot,
174    #[token("%")]
175    Percent,
176    #[token("==")]
177    EqEq,
178    #[token("!=")]
179    NotEq,
180    #[token("<=")]
181    LtEq,
182    #[token(">=")]
183    GtEq,
184    #[token("~")]
185    Tilde,
186    #[token("^")]
187    Hat,
188    #[token("**")]
189    StarStar,
190    #[token("**=")]
191    StarStarEq,
192    #[token("+=")]
193    PlusEq,
194    #[token("-=")]
195    MinusEq,
196    #[token("*=")]
197    StarEq,
198    #[token("/=")]
199    SlashEq,
200    #[token("%=")]
201    PercentEq,
202    #[token("&=")]
203    AmperEq,
204    #[token("|=")]
205    PipeEq,
206    #[token("^=")]
207    HatEq,
208    #[token("<<=")]
209    LtLtEq,
210    #[token(">>=")]
211    GtGtEq,
212    #[token("->")]
213    Arrow,
214    #[token("=>")]
215    FatArrow,
216}
217
218impl TokenKind {
219    /// Return a user-friendly description of the token kind. E.g.
220    /// TokenKind::Newline => "a newline"
221    /// TokenKind::Colon => "`:`"
222    pub fn describe(&self) -> &str {
223        use TokenKind::*;
224        match self {
225            Newline => "a newline",
226            Name => "a name",
227            Int => "a number",
228            Hex => "a hexadecimal number",
229            Octal => "an octal number",
230            Binary => "a binary number",
231            Text => "a string",
232
233            True => "keyword `true`",
234            False => "keyword `false`",
235            Assert => "keyword `assert`",
236            Break => "keyword `break`",
237            Continue => "keyword `continue`",
238            Contract => "keyword `contract`",
239            Fn => "keyword `fn`",
240            Const => "keyword `const`",
241            Let => "keyword `let`",
242            Mut => "keyword `mut`",
243            Else => "keyword `else`",
244            Idx => "keyword `idx`",
245            If => "keyword `if`",
246            Match => "keyword `match`",
247            Impl => "keyword `impl`",
248            Pragma => "keyword `pragma`",
249            For => "keyword `for`",
250            Pub => "keyword `pub`",
251            Return => "keyword `return`",
252            Revert => "keyword `revert`",
253            SelfType => "keyword `Self`",
254            SelfValue => "keyword `self`",
255            Struct => "keyword `struct`",
256            Enum => "keyword `enum`",
257            Trait => "keyword `trait`",
258            Type => "keyword `type`",
259            Unsafe => "keyword `unsafe`",
260            While => "keyword `while`",
261            And => "keyword `and`",
262            As => "keyword `as`",
263            In => "keyword `in`",
264            Not => "keyword `not`",
265            Or => "keyword `or`",
266            Use => "keyword `use`",
267            ParenOpen => "symbol `(`",
268            ParenClose => "symbol `)`",
269            BracketOpen => "symbol `[`",
270            BracketClose => "symbol `]`",
271            BraceOpen => "symbol `{`",
272            BraceClose => "symbol `}`",
273            Colon => "symbol `:`",
274            ColonColon => "symbol `::`",
275            Comma => "symbol `,`",
276            Hash => "symbol `#`",
277            Semi => "symbol `;`",
278            Plus => "symbol `+`",
279            Minus => "symbol `-`",
280            Star => "symbol `*`",
281            Slash => "symbol `/`",
282            Pipe => "symbol `|`",
283            Amper => "symbol `&`",
284            Lt => "symbol `<`",
285            LtLt => "symbol `<<`",
286            Gt => "symbol `>`",
287            GtGt => "symbol `>>`",
288            Eq => "symbol `=`",
289            Dot => "symbol `.`",
290            DotDot => "symbol `..`",
291            Percent => "symbol `%`",
292            EqEq => "symbol `==`",
293            NotEq => "symbol `!=`",
294            LtEq => "symbol `<=`",
295            GtEq => "symbol `>=`",
296            Tilde => "symbol `~`",
297            Hat => "symbol `^`",
298            StarStar => "symbol `**`",
299            StarStarEq => "symbol `**=`",
300            PlusEq => "symbol `+=`",
301            MinusEq => "symbol `-=`",
302            StarEq => "symbol `*=`",
303            SlashEq => "symbol `/=`",
304            PercentEq => "symbol `%=`",
305            AmperEq => "symbol `&=`",
306            PipeEq => "symbol `|=`",
307            HatEq => "symbol `^=`",
308            LtLtEq => "symbol `<<=`",
309            GtGtEq => "symbol `>>=`",
310            Arrow => "symbol `->`",
311            FatArrow => "symbol `=>`",
312
313            Error => unreachable!(), // TODO this is reachable
314        }
315    }
316}
fe_parser/lexer/token.rs

fe_parser/lexer/
token.rs
