Basic integer support

src/tokenizer.rs

@@ -6,6 +6,7 @@ pub enum Token {
 	CloseParen,
 	Identifier(String),
 	String(String),
+	Integer(i32)
 }
 
 
@@ -36,6 +37,8 @@ pub struct Tokenizer {
 	reading_string: bool,
 	escape_next_char: bool,
 
+	reading_num: bool,
+
 	reading_identifier: bool,
 	skipping_comment: bool,
 
@@ -49,6 +52,8 @@ impl Tokenizer {
 			line: 1,
 			column: 1,
 
+			reading_num: false,
+			
 			reading_string: false,
 			escape_next_char: false,
 
@@ -93,6 +98,16 @@ impl Tokenizer {
 				continue;
 			}
 
+			if self.reading_num {
+				// Allow spacing numbers like 1_000_000
+				if !char.is_numeric() && char != '_'  {
+					self.reading_num = false;
+
+					tokens.push(Token::Integer(self.storage.iter().collect::<String>().parse().unwrap()));
+					self.storage.clear();
+				}
+			}
+
 			if self.reading_string {
 				if char == '"' && !self.escape_next_char {
 					self.reading_string = false;
@@ -138,9 +153,16 @@ impl Tokenizer {
 						self.storage.clear();
 						self.storage.push(c);
 						continue;
-					}
 
-					print!("{}", char)
+					// Allow numbers to also start with _ for fun
+					// ______100_0__: 1,000
+					// TODO: delete this when I expand identifiers to include more symbols
+					} else if c.is_numeric() || c == '_' {
+						self.reading_num = true;
+						self.storage.clear();
+						self.storage.push(c);
+						continue;
+					}
 				}
 			}
 		}