fe_parser/
lexer.rs

1mod token;
2use crate::node::Span;
3use fe_common::files::SourceFileId;
4use logos::Logos;
5pub use token::{Token, TokenKind};
6
7#[derive(Clone)]
8pub struct Lexer<'a> {
9    file_id: SourceFileId,
10    inner: logos::Lexer<'a, TokenKind>,
11}
12
13impl<'a> Lexer<'a> {
14    /// Create a new lexer with the given source code string.
15    pub fn new(file_id: SourceFileId, src: &'a str) -> Lexer {
16        Lexer {
17            file_id,
18            inner: TokenKind::lexer(src),
19        }
20    }
21
22    /// Return the full source code string that's being tokenized.
23    pub fn source(&self) -> &'a str {
24        self.inner.source()
25    }
26}
27
28impl<'a> Iterator for Lexer<'a> {
29    type Item = Token<'a>;
30
31    fn next(&mut self) -> Option<Self::Item> {
32        let kind = self.inner.next()?;
33        let text = self.inner.slice();
34        let span = self.inner.span();
35        Some(Token {
36            kind,
37            text,
38            span: Span {
39                file_id: self.file_id,
40                start: span.start,
41                end: span.end,
42            },
43        })
44    }
45}
46
47#[cfg(test)]
48mod tests {
49    use crate::lexer::{Lexer, TokenKind};
50    use fe_common::files::SourceFileId;
51    use TokenKind::*;
52
53    fn check(input: &str, expected: &[TokenKind]) {
54        let lex = Lexer::new(SourceFileId::dummy_file(), input);
55
56        let actual = lex.map(|t| t.kind).collect::<Vec<_>>();
57
58        assert!(
59            actual.iter().eq(expected.iter()),
60            "\nexpected: {expected:?}\n  actual: {actual:?}"
61        );
62    }
63
64    #[test]
65    fn basic() {
66        check(
67            "contract Foo:\n  x: u32\n  fn f() -> u32: not x",
68            &[
69                Contract, Name, Colon, Newline, Name, Colon, Name, Newline, Fn, Name, ParenOpen,
70                ParenClose, Arrow, Name, Colon, Not, Name,
71            ],
72        );
73    }
74
75    #[test]
76    fn strings() {
77        let rawstr = r#""string \t with \n escapes \" \"""#;
78        let mut lex = Lexer::new(SourceFileId::dummy_file(), rawstr);
79        let lexedstr = lex.next().unwrap();
80        assert!(lexedstr.kind == Text);
81        assert!(lexedstr.text == rawstr);
82        assert!(lex.next().is_none());
83    }
84
85    #[test]
86    fn errors() {
87        check(
88            "contract Foo@ 5u8 \n  self.bar",
89            &[
90                Contract, Name, Error, Int, Name, Newline, SelfValue, Dot, Name,
91            ],
92        );
93    }
94
95    #[test]
96    fn tabs_and_comment() {
97        check(
98            "\n\t \tcontract\n\tFoo // hi mom!\n ",
99            &[Newline, Contract, Newline, Name, Newline],
100        );
101    }
102}
fe_parser/lexer.rs

fe_parser/
lexer.rs
