Improvements to lexer, add test and benchmarks to lexer

- Change lexer functions to return a option
- Return None variant instead of \u0000 in case of EOF
- Test lexer against a sample program

1 files changed, 2 insertions, 1 deletions

diff --git a/src/main.rs b/src/main.rs
index fa355c9..8f3c001 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,3 +1,4 @@
+#![feature(test, let_chains)]
 #![warn(
     clippy::wildcard_imports,
     clippy::string_add,
@@ -24,6 +25,6 @@ if variable >= 7 {
 fn main() {
     let lexer = Lexer::new(SOURCE);
     for token in lexer {
-        print!("({}) ", token.lexeme);
+        print!("{} ", token.lexeme);
     }
 }