diff --git a/src/tokenizer.rs b/src/tokenizer.rs index dfe80b7..c09ec19 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -6,6 +6,7 @@ pub enum Token { CloseParen, Identifier(String), String(String), + Integer(i32) } @@ -36,6 +37,8 @@ pub struct Tokenizer { reading_string: bool, escape_next_char: bool, + reading_num: bool, + reading_identifier: bool, skipping_comment: bool, @@ -49,6 +52,8 @@ impl Tokenizer { line: 1, column: 1, + reading_num: false, + reading_string: false, escape_next_char: false, @@ -93,6 +98,16 @@ impl Tokenizer { continue; } + if self.reading_num { + // Allow spacing numbers like 1_000_000 + if !char.is_numeric() && char != '_' { + self.reading_num = false; + + tokens.push(Token::Integer(self.storage.iter().collect::().parse().unwrap())); + self.storage.clear(); + } + } + if self.reading_string { if char == '"' && !self.escape_next_char { self.reading_string = false; @@ -138,9 +153,16 @@ impl Tokenizer { self.storage.clear(); self.storage.push(c); continue; - } - print!("{}", char) + // Allow numbers to also start with _ for fun + // ______100_0__: 1,000 + // TODO: delete this when I expand identifiers to include more symbols + } else if c.is_numeric() || c == '_' { + self.reading_num = true; + self.storage.clear(); + self.storage.push(c); + continue; + } } } }