add tests to scanner.

move chunk out of lib.rs
add -f and -s and -c flags to main

Cargo.lock

@@ -101,10 +101,17 @@ name = "mox"
 version = "0.1.0"
 dependencies = [
  "clap",
+ "peekmore",
  "strum",
  "strum_macros",
 ]
 
+[[package]]
+name = "peekmore"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9163e1259760e83d528d1b3171e5100c1767f10c52e1c4d6afad26e63d47d758"
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.81"

Cargo.toml

@@ -7,5 +7,6 @@ edition = "2021"
 
 [dependencies]
 clap = "4.5.4"
+peekmore = "1.3.0"
 strum = "0.26"
 strum_macros = "0.26"

src/chunk.rs

@@ -0,0 +1,112 @@
+use std::u16;
+
+use crate::value::Value;
+
+#[derive(Debug, PartialEq, Eq, strum_macros::Display)]
+pub enum OptCode {
+    OpReturn,
+    OpConstant,
+    OpNegate,
+    OpAdd,
+    OpSubstract,
+    OpMultiply,
+    OpDivide,
+}
+#[derive(Debug)]
+pub struct Chunk {
+    pub code: Vec<u8>,
+    pub constants: Vec<Value>,
+    pub lines: Vec<u16>,
+}
+
+impl From<OptCode> for u8 {
+    fn from(code: OptCode) -> Self {
+        match code {
+            OptCode::OpReturn => 0,
+            OptCode::OpConstant => 1,
+            OptCode::OpNegate => 2,
+            OptCode::OpAdd => 3,
+            OptCode::OpSubstract => 4,
+            OptCode::OpMultiply => 5,
+            OptCode::OpDivide => 6,
+        }
+    }
+}
+#[derive(Debug)]
+pub struct ConversionError {
+    pub invalid_value: u8,
+    pub message: String,
+}
+
+impl ConversionError {
+    fn new(invalid_value: u8) -> Self {
+        ConversionError {
+            invalid_value,
+            message: format!("Invalid opcode"),
+        }
+    }
+}
+
+impl TryFrom<u8> for OptCode {
+    type Error = ConversionError;
+    fn try_from(code: u8) -> Result<Self, Self::Error> {
+        match code {
+            0 => Ok(OptCode::OpReturn),
+            1 => Ok(OptCode::OpConstant),
+            2 => Ok(OptCode::OpNegate),
+            3 => Ok(OptCode::OpAdd),
+            4 => Ok(OptCode::OpSubstract),
+            5 => Ok(OptCode::OpMultiply),
+            6 => Ok(OptCode::OpDivide),
+            _ => Err(ConversionError::new(code)),
+        }
+    }
+}
+
+impl Chunk {
+    pub fn new() -> Self {
+        Self {
+            code: vec![],
+            constants: vec![],
+            lines: vec![],
+        }
+    }
+    pub fn write(&mut self, byte: u8, line: u16) {
+        self.code.push(byte);
+        self.lines.push(line);
+    }
+    pub fn write_value(&mut self, add: Value) -> u8 {
+        self.constants.push(add);
+        return (self.constants.len() - 1).try_into().unwrap();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn u8_to_optcode() {
+        let pred: OptCode = 0.try_into().unwrap();
+        assert_eq!(pred, OptCode::OpReturn);
+        let pred: OptCode = 1.try_into().unwrap();
+        assert_eq!(pred, OptCode::OpConstant);
+    }
+    #[test]
+    fn basic_write() {
+        let mut chunk = Chunk::new();
+        let constant_idx: u8 = chunk.write_value(1.2);
+        assert_eq!(constant_idx, 0);
+        chunk.write(OptCode::OpConstant.into(), 123);
+        chunk.write(constant_idx, 123);
+        let pred: OptCode = chunk.code[0].try_into().unwrap();
+        assert_eq!(pred, OptCode::OpConstant);
+    }
+    #[test]
+    fn try_into_constant() {
+        let mut chunk = Chunk::new();
+        let constant_idx: u8 = chunk.write_value(1.2);
+        chunk.write(OptCode::OpConstant.into(), 123);
+        chunk.write(constant_idx, 123);
+    }
+}

src/compiler.rs

@@ -1,26 +1,22 @@
-use crate::scanner::Scanner;
-use crate::scanner::TokenType;
+use strum_macros::Display;
 
-pub fn compile(source: &str) {
-    let mut scanner: Scanner = Scanner::new(source);
-    let mut line = 0;
-    loop {
-        let token = scanner.scan_token();
+use crate::chunk::Chunk;
 
-        if token.line != line {
-            print!("{:4}", token.line);
-            line = token.line;
-        } else {
-            print!("   |");
-        }
-
-        println!(
-            ":{:<3} {:20} {:20}",
-            token.start, token.token_type, token.lexeme
-        );
-
-        if let TokenType::TokenEof = token.token_type {
-            break;
-        }
-    }
+pub fn compile(source: &str, chunk: &Chunk) -> bool {
+    return true;
+}
+
+#[derive(Display, PartialEq, Eq, PartialOrd, Ord)]
+enum Precedence {
+    None,
+    Assignment,
+    Or,
+    And,
+    Equality,
+    Comparison,
+    Term,
+    Factor,
+    Unary,
+    Call,
+    Primary,
 }

src/debug.rs

@@ -1,5 +1,5 @@
+use crate::chunk::{Chunk, ConversionError, OptCode};
 use crate::value::Value;
-use crate::{Chunk, ConversionError, OptCode};
 use std::env;
 
 pub fn trace_enabled() -> bool {

src/lib.rs

@@ -1,118 +1,6 @@
+pub mod chunk;
 pub mod compiler;
 pub mod debug;
 pub mod scanner;
 pub mod value;
 pub mod vm;
-
-use std::u16;
-
-use value::Value;
-
-#[derive(Debug, PartialEq, Eq, strum_macros::Display)]
-pub enum OptCode {
-    OpReturn,
-    OpConstant,
-    OpNegate,
-    OpAdd,
-    OpSubstract,
-    OpMultiply,
-    OpDivide,
-}
-#[derive(Debug)]
-pub struct Chunk {
-    pub code: Vec<u8>,
-    pub constants: Vec<Value>,
-    pub lines: Vec<u16>,
-}
-
-impl From<OptCode> for u8 {
-    fn from(code: OptCode) -> Self {
-        match code {
-            OptCode::OpReturn => 0,
-            OptCode::OpConstant => 1,
-            OptCode::OpNegate => 2,
-            OptCode::OpAdd => 3,
-            OptCode::OpSubstract => 4,
-            OptCode::OpMultiply => 5,
-            OptCode::OpDivide => 6,
-        }
-    }
-}
-#[derive(Debug)]
-pub struct ConversionError {
-    invalid_value: u8,
-    message: String,
-}
-
-impl ConversionError {
-    fn new(invalid_value: u8) -> Self {
-        ConversionError {
-            invalid_value,
-            message: format!("Invalid opcode"),
-        }
-    }
-}
-
-impl TryFrom<u8> for OptCode {
-    type Error = ConversionError;
-    fn try_from(code: u8) -> Result<Self, Self::Error> {
-        match code {
-            0 => Ok(OptCode::OpReturn),
-            1 => Ok(OptCode::OpConstant),
-            2 => Ok(OptCode::OpNegate),
-            3 => Ok(OptCode::OpAdd),
-            4 => Ok(OptCode::OpSubstract),
-            5 => Ok(OptCode::OpMultiply),
-            6 => Ok(OptCode::OpDivide),
-            _ => Err(ConversionError::new(code)),
-        }
-    }
-}
-
-impl Chunk {
-    pub fn new() -> Self {
-        Self {
-            code: vec![],
-            constants: vec![],
-            lines: vec![],
-        }
-    }
-    pub fn write(&mut self, byte: u8, line: u16) {
-        self.code.push(byte);
-        self.lines.push(line);
-    }
-    pub fn write_value(&mut self, add: Value) -> u8 {
-        self.constants.push(add);
-        return (self.constants.len() - 1).try_into().unwrap();
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn u8_to_optcode() {
-        let pred: OptCode = 0.try_into().unwrap();
-        assert_eq!(pred, OptCode::OpReturn);
-        let pred: OptCode = 1.try_into().unwrap();
-        assert_eq!(pred, OptCode::OpConstant);
-    }
-    #[test]
-    fn basic_write() {
-        let mut chunk = Chunk::new();
-        let constant_idx: u8 = chunk.write_value(1.2);
-        assert_eq!(constant_idx, 0);
-        chunk.write(OptCode::OpConstant.into(), 123);
-        chunk.write(constant_idx, 123);
-        let pred: OptCode = chunk.code[0].try_into().unwrap();
-        assert_eq!(pred, OptCode::OpConstant);
-    }
-    #[test]
-    fn try_into_constant() {
-        let mut chunk = Chunk::new();
-        let constant_idx: u8 = chunk.write_value(1.2);
-        chunk.write(OptCode::OpConstant.into(), 123);
-        chunk.write(constant_idx, 123);
-    }
-}

src/main.rs

@@ -1,9 +1,11 @@
 use clap::{Arg, ArgAction, Command};
+use mox::chunk::Chunk;
+use mox::scanner::Scanner;
 use mox::vm::InterpretResult;
 use mox::vm::VM;
 use std::fs;
-use std::io;
-use std::io::Write;
+// use std::io;
+// use std::io::Write;
 use std::process;
 use std::process::exit;
 
@@ -27,59 +29,74 @@ fn main() {
                 .action(ArgAction::Set)
                 .value_name("COMMAND")
                 .help("command to run"),
+        )
+        .arg(
+            Arg::new("scan")
+                .short('s')
+                .long("scan")
+                .action(ArgAction::SetTrue)
+                .value_name("SCAN")
+                .help("scan only"),
         );
 
     let matches = app.get_matches();
 
-    if let Some(command) = matches.get_one::<String>("command") {
-        run_content(command);
-    } else if let Some(file) = matches.get_one::<String>("file") {
-        run_file(file);
-    } else {
-        repl();
-    }
-}
-
-fn repl() {
-    let mut input = String::new();
-    let mut vm: VM = VM::new();
-    loop {
-        input.clear();
-        print!("> ");
-        io::stdout().flush().unwrap();
-
-        match io::stdin().read_line(&mut input) {
-            Ok(bytes) => {
-                if bytes == 0 || input.trim() == "exit" {
-                    println!("Bye!");
-                    break;
-                }
-                vm.interpret(&input);
-            }
-            Err(_error) => {
-                continue;
-            }
+    if let Some(source) = matches.get_one::<String>("command") {
+        if let Some(_) = matches.get_one::<bool>("scan") {
+            scan_content(source);
+        } else {
+            run_content(source);
         }
+    } else if let Some(file) = matches.get_one::<String>("file") {
+        let source = fs::read_to_string(file).unwrap_or_else(|err| {
+            eprintln!("Error reading '{}': {}", file, err);
+            process::exit(74);
+        });
+        println!("{}", source);
+        if let Some(_) = matches.get_one::<bool>("scan") {
+            scan_content(&source);
+        } else {
+            run_content(&source);
+        }
+    } else {
+        todo!("repl not done yet")
+        // repl();
     }
 }
 
+// fn repl() {
+//     let mut input = String::new();
+//     let mut vm: VM = VM::new();
+//     let mut chunk: Chunk = Chunk::new();
+//     loop {
+//         input.clear();
+//         print!("> ");
+//         io::stdout().flush().unwrap();
+//
+//         match io::stdin().read_line(&mut input) {
+//             Ok(bytes) => {
+//                 if bytes == 0 || input.trim() == "exit" {
+//                     println!("Bye!");
+//                     break;
+//                 }
+//                 vm.interpret(&input, &mut chunk);
+//             }
+//             Err(_error) => {
+//                 continue;
+//             }
+//         }
+//     }
+// }
+
+fn scan_content(source: &str) {
+    let input = source.to_owned();
+    let mut scanner = Scanner::new(&input);
+    scanner.compile();
+}
 fn run_content(source: &str) {
     let mut vm: VM = VM::new();
-    match vm.interpret(source) {
-        InterpretResult::InterpretOk => exit(0),
-        InterpretResult::InterpretCompileError => exit(65),
-        InterpretResult::InterpretRuntimeError => exit(70),
-    }
-}
-
-fn run_file(path: &str) {
-    let mut vm: VM = VM::new();
-    let content = fs::read_to_string(path).unwrap_or_else(|err| {
-        eprintln!("Error reading '{}': {}", path, err);
-        process::exit(74);
-    });
-
-    match vm.interpret(&content) {
+    let mut chunk: Chunk = Chunk::new();
+    match vm.interpret(source, &mut chunk) {
         InterpretResult::InterpretOk => exit(0),
         InterpretResult::InterpretCompileError => exit(65),
         InterpretResult::InterpretRuntimeError => exit(70),

src/scanner.rs

@@ -1,10 +1,13 @@
+use peekmore::{PeekMore, PeekMoreIterator};
 use std::fmt;
+use std::str::Chars;
 
 pub struct Scanner<'a> {
+    source: &'a String,
     start: usize,
+    chars: PeekMoreIterator<Chars<'a>>,
     current: usize,
     line: u16,
-    source: &'a str,
 }
 
 impl<'a> fmt::Display for Scanner<'a> {
@@ -21,147 +24,149 @@ impl<'a> fmt::Display for Scanner<'a> {
 }
 
 impl<'a> Scanner<'a> {
-    pub fn new(source: &'a str) -> Self {
+    pub fn new(source: &'a String) -> Self {
         Scanner {
             start: 0,
             current: 0,
             line: 1,
+            chars: source.chars().peekmore(),
             source,
         }
     }
+    pub fn compile(&mut self) {
+        let mut line = 0;
+        loop {
+            let token = self.scan_token();
+
+            if token.line != line {
+                print!("{:4}", token.line);
+                line = token.line;
+            } else {
+                print!("   |");
+            }
+
+            println!(
+                ":{:<3} {:20} {:20}",
+                token.start, token.token_type, token.lexeme
+            );
+
+            if let TokenType::TokenEof = token.token_type {
+                break;
+            }
+        }
+    }
     pub fn scan_token(&mut self) -> Token {
         self.skip_whitespace();
         self.start = self.current;
-        if self.is_at_end() {
-            return self.make_token(TokenType::TokenEof);
-        }
-        let c: char = self.advance();
-
-        match c {
-            c if c.is_digit(10) => return self.make_number(),
-            c if c.is_alphabetic() || c == '_' => return self.make_identifier(),
-            '(' => return self.make_token(TokenType::TokenLeftParen),
-            ')' => return self.make_token(TokenType::TokenRightParen),
-            '{' => return self.make_token(TokenType::TokenLeftBrace),
-            '}' => return self.make_token(TokenType::TokenRightBrace),
-            ';' => return self.make_token(TokenType::TokenSemicolon),
-            ',' => return self.make_token(TokenType::TokenComma),
-            '.' => return self.make_token(TokenType::TokenDot),
-            '-' => return self.make_token(TokenType::TokenMinus),
-            '+' => return self.make_token(TokenType::TokenPlus),
-            '/' => return self.make_token(TokenType::TokenSlash),
-            '*' => return self.make_token(TokenType::TokenStar),
-            '!' => {
-                if self.matches('=') {
-                    return self.make_token(TokenType::TokenBangEqual);
-                } else {
-                    return self.make_token(TokenType::TokenBang);
+        match self.advance() {
+            Some(c) => match c {
+                c if c.is_digit(10) => return self.make_number(),
+                c if c.is_ascii_alphabetic() || c == '_' => return self.make_identifier(),
+                '(' => return self.make_token(TokenType::TokenLeftParen),
+                ')' => return self.make_token(TokenType::TokenRightParen),
+                '{' => return self.make_token(TokenType::TokenLeftBrace),
+                '}' => return self.make_token(TokenType::TokenRightBrace),
+                ';' => return self.make_token(TokenType::TokenSemicolon),
+                ',' => return self.make_token(TokenType::TokenComma),
+                '.' => return self.make_token(TokenType::TokenDot),
+                '-' => return self.make_token(TokenType::TokenMinus),
+                '+' => return self.make_token(TokenType::TokenPlus),
+                '/' => return self.make_token(TokenType::TokenSlash),
+                '*' => return self.make_token(TokenType::TokenStar),
+                '!' => {
+                    return self.make_token_if_matches(
+                        '=',
+                        TokenType::TokenBangEqual,
+                        TokenType::TokenBang,
+                    )
                 }
-            }
-            '=' => {
-                if self.matches('=') {
-                    return self.make_token(TokenType::TokenEqualEqual);
-                } else {
-                    return self.make_token(TokenType::TokenEqual);
+                '=' => {
+                    return self.make_token_if_matches(
+                        '=',
+                        TokenType::TokenEqualEqual,
+                        TokenType::TokenEqual,
+                    )
                 }
-            }
-            '<' => {
-                if self.matches('=') {
-                    return self.make_token(TokenType::TokenLessEqual);
-                } else {
-                    return self.make_token(TokenType::TokenLess);
+                '<' => {
+                    return self.make_token_if_matches(
+                        '=',
+                        TokenType::TokenLessEqual,
+                        TokenType::TokenLess,
+                    )
                 }
-            }
-            '>' => {
-                if self.matches('=') {
-                    return self.make_token(TokenType::TokenGreaterEqual);
-                } else {
-                    return self.make_token(TokenType::TokenGreater);
+                '>' => {
+                    return self.make_token_if_matches(
+                        '=',
+                        TokenType::TokenGreaterEqual,
+                        TokenType::TokenGreater,
+                    )
                 }
-            }
-            '"' => return self.make_string(),
-            _ => return self.make_error_token("Unexpected character."),
+                '"' => return self.make_string(),
+                _ => return self.make_error_token("Unexpected character."),
+            },
+            None => return self.make_eof(),
         };
     }
     fn make_identifier(&mut self) -> Token {
-        while let Some(c) = self.peek() {
-            if c.is_alphabetic() || c == '_' || c.is_digit(10) {
+        while let Some(c) = self.chars.peek() {
+            if c.is_alphabetic() || *c == '_' || c.is_digit(10) {
                 self.advance();
             } else {
                 break;
             }
         }
-        return self.make_token(self.identifier_type());
-    }
-    fn identifier_type(&self) -> TokenType {
-        let c = self.source.chars().nth(self.start);
+        let lexeme = &self.source[self.start..self.current];
 
-        match c {
-            Some('a') => return self.check_keyword(1, 2, "nd", TokenType::TokenAnd),
-            Some('c') if self.current - self.start > 1 => {
-                match self.source.chars().nth(self.start + 1) {
-                    Some('l') => return self.check_keyword(2, 3, "ass", TokenType::TokenClass),
-                    Some('a') => return self.check_keyword(2, 1, "p", TokenType::TokenFalse),
-                    _ => panic!("bad keyword"),
-                }
-            }
-            Some('e') => return self.check_keyword(1, 3, "lse", TokenType::TokenElse),
-            Some('i') => return self.check_keyword(1, 1, "f", TokenType::TokenIf),
-            Some('n') if self.current - self.start > 1 => {
-                match self.source.chars().nth(self.start + 1) {
-                    Some('i') => return self.check_keyword(2, 1, "l", TokenType::TokenNil),
-                    Some('o') => return self.check_keyword(2, 0, "", TokenType::TokenFalse),
-                    _ => panic!("bad keyword"),
-                }
-            }
-            Some('o') => return self.check_keyword(1, 1, "r", TokenType::TokenOr),
-            Some('p') => return self.check_keyword(1, 4, "rint", TokenType::TokenPrint),
-            Some('r') => return self.check_keyword(1, 5, "eturn", TokenType::TokenReturn),
-            Some('s') => return self.check_keyword(1, 4, "uper", TokenType::TokenSuper),
-            Some('v') => return self.check_keyword(1, 2, "ar", TokenType::TokenVar),
-            Some('w') => return self.check_keyword(1, 4, "hile", TokenType::TokenWhile),
-            Some('f') if self.current - self.start > 1 => {
-                match self.source.chars().nth(self.start + 1) {
-                    Some('a') => return self.check_keyword(2, 3, "lse", TokenType::TokenFalse),
-                    Some('o') => return self.check_keyword(2, 1, "r", TokenType::TokenFor),
-                    Some('u') => return self.check_keyword(2, 1, "n", TokenType::TokenFun),
-                    _ => panic!("bad keyword"),
-                }
-            }
-            Some('t') if self.current - self.start > 1 => {
-                match self.source.chars().nth(self.start + 1) {
-                    Some('h') => return self.check_keyword(2, 2, "is", TokenType::TokenThis),
-                    Some('r') => return self.check_keyword(2, 2, "ue", TokenType::TokenTrue),
-                    _ => panic!("bad keyword"),
-                }
-            }
-            _ => return TokenType::TokenIdentifier,
-        };
+        match lexeme {
+            "and" => self.make_token(TokenType::TokenAnd),
+            "class" => self.make_token(TokenType::TokenClass),
+            "cap" => self.make_token(TokenType::TokenFalse),
+            "else" => self.make_token(TokenType::TokenElse),
+            "if" => self.make_token(TokenType::TokenIf),
+            "nil" => self.make_token(TokenType::TokenNil),
+            "no" => self.make_token(TokenType::TokenFalse),
+            "or" => self.make_token(TokenType::TokenOr),
+            "print" => self.make_token(TokenType::TokenPrint),
+            "return" => self.make_token(TokenType::TokenReturn),
+            "super" => self.make_token(TokenType::TokenSuper),
+            "var" => self.make_token(TokenType::TokenVar),
+            "while" => self.make_token(TokenType::TokenWhile),
+            "false" => self.make_token(TokenType::TokenFalse),
+            "for" => self.make_token(TokenType::TokenFor),
+            "fun" => self.make_token(TokenType::TokenFun),
+            "this" => self.make_token(TokenType::TokenThis),
+            "true" => self.make_token(TokenType::TokenTrue),
+            _ => return self.make_token(TokenType::TokenIdentifier),
+        }
     }
-    fn check_keyword(
-        &self,
-        start: usize,
-        length: usize,
-        rest: &str,
-        token_type: TokenType,
-    ) -> TokenType {
-        let end = self.start + start + length;
-        let s = &self.source[self.start + start..end];
-        let next = self.source.chars().nth(end);
-        if (self.current == end) && (s == rest) {
-            match next {
-                Some(n) if n.is_whitespace() => {
-                    return token_type;
-                }
-                _ => return TokenType::TokenIdentifier,
-            }
+    fn make_token_if_matches(
+        &mut self,
+        expected: char,
+        on_match: TokenType,
+        otherwise: TokenType,
+    ) -> Token {
+        if self.matches(expected) {
+            self.make_token(on_match)
         } else {
-            return TokenType::TokenIdentifier;
+            self.make_token(otherwise)
+        }
+    }
+    fn matches(&mut self, expected: char) -> bool {
+        match self.chars.peek() {
+            Some(c) => {
+                if c == &expected {
+                    self.advance();
+                    true
+                } else {
+                    false
+                }
+            }
+            None => false,
         }
     }
     fn make_string(&mut self) -> Token {
         loop {
-            match self.peek() {
+            match self.chars.peek() {
                 Some(c) => match c {
                     '\n' => {
                         self.line += 1;
@@ -216,8 +221,7 @@ impl<'a> Scanner<'a> {
     }
     fn skip_whitespace(&mut self) {
         loop {
-            let peek = self.peek();
-            match peek {
+            match self.chars.peek() {
                 None => return,
                 Some(c) => match c {
                     ' ' | '\r' | '\t' => {
@@ -229,10 +233,14 @@ impl<'a> Scanner<'a> {
                         self.advance();
                         continue;
                     }
-                    '/' => match self.peek_next() {
-                        Some(c) if c == '/' => {
-                            while let Some(_) = self.peek() {
-                                if !self.is_at_end() {
+                    '/' => match self.chars.peek_nth(1) {
+                        Some(c) if *c == '/' => {
+                            while let Some(c) = self.peek() {
+                                if c == '\n' {
+                                    self.line += 1;
+                                    self.advance();
+                                    return;
+                                } else {
                                     self.advance();
                                 }
                             }
@@ -255,28 +263,9 @@ impl<'a> Scanner<'a> {
             line: self.line,
         }
     }
-    fn advance(&mut self) -> char {
+    fn advance(&mut self) -> Option<char> {
         self.current += 1;
-
-        match self.source.chars().nth(self.current - 1) {
-            Some(c) => c,
-            None => panic!(
-                "advance failed: fell off the end. {}:{}",
-                self.line, self.current
-            ),
-        }
-    }
-    fn matches(&mut self, expected: char) -> bool {
-        match self.source.chars().nth(self.current) {
-            Some(c) => {
-                if c != expected {
-                    return false;
-                }
-                self.current += 1;
-                return true;
-            }
-            None => return false,
-        }
+        self.chars.next()
     }
     fn make_error_token(&self, message: &str) -> Token {
         Token {
@@ -286,6 +275,14 @@ impl<'a> Scanner<'a> {
             line: self.line,
         }
     }
+    fn make_eof(&self) -> Token {
+        Token {
+            token_type: TokenType::TokenEof,
+            start: self.start,
+            lexeme: "".to_string(),
+            line: self.line,
+        }
+    }
 }
 
 pub struct Token {
@@ -295,9 +292,7 @@ pub struct Token {
     pub line: u16,
 }
 
-impl Token {}
-
-#[derive(Clone, Copy, strum_macros::Display)]
+#[derive(Clone, Copy, strum_macros::Display, Debug, PartialEq, Eq)]
 pub enum TokenType {
     // Single-character tokens.
     TokenLeftParen,
@@ -345,3 +340,86 @@ pub enum TokenType {
     TokenError,
     TokenEof,
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::scanner;
+    use crate::scanner::TokenType;
+    #[test]
+    fn single_chars() {
+        assert_token(String::from(""), TokenType::TokenEof);
+        assert_token(String::from("("), TokenType::TokenLeftParen);
+        assert_token(String::from("}"), TokenType::TokenRightBrace);
+        assert_token(String::from("-"), TokenType::TokenMinus);
+        assert_token(String::from("+"), TokenType::TokenPlus);
+        assert_token(String::from("/"), TokenType::TokenSlash);
+    }
+
+    #[test]
+    fn double_chars() {
+        assert_token(String::from("=="), TokenType::TokenEqualEqual);
+        assert_token(String::from("!="), TokenType::TokenBangEqual);
+        assert_token(String::from(">"), TokenType::TokenGreater);
+        assert_token(String::from(">="), TokenType::TokenGreaterEqual);
+    }
+    #[test]
+    fn strings() {
+        assert_token_lexeme(String::from("\"mox\""), TokenType::TokenString, "\"mox\"");
+        assert_token_lexeme(String::from("\"\""), TokenType::TokenString, "\"\"");
+    }
+
+    #[test]
+    fn numbers() {
+        assert_token_lexeme(String::from("0"), TokenType::TokenNumber, "0");
+        assert_token_lexeme(String::from("4"), TokenType::TokenNumber, "4");
+        assert_token_lexeme(String::from("42"), TokenType::TokenNumber, "42");
+        assert_token_lexeme(String::from("13.99"), TokenType::TokenNumber, "13.99");
+    }
+    #[test]
+    fn newlines() {
+        assert_tokens(
+            String::from("+\n//comment\n-"),
+            &vec![TokenType::TokenPlus, TokenType::TokenMinus],
+        );
+    }
+
+    #[test]
+    fn identifier() {
+        assert_token(String::from("class"), TokenType::TokenClass);
+        assert_token(String::from("if"), TokenType::TokenIf);
+        assert_token(String::from("while"), TokenType::TokenWhile);
+        assert_token(String::from("true"), TokenType::TokenTrue);
+        assert_token(String::from("false"), TokenType::TokenFalse);
+
+        assert_token(String::from("cap"), TokenType::TokenFalse);
+
+        assert_token_lexeme(String::from("mox"), TokenType::TokenIdentifier, "mox");
+    }
+    fn assert_token(source: String, expected: scanner::TokenType) {
+        let mut scanner = scanner::Scanner::new(&source);
+        let token = scanner.scan_token();
+
+        assert_eq!(token.token_type, expected);
+    }
+
+    fn assert_token_lexeme(
+        source: String,
+        expected_type: scanner::TokenType,
+        expected_lexeme: &str,
+    ) {
+        let mut scanner = scanner::Scanner::new(&source);
+        let token = scanner.scan_token();
+
+        assert_eq!(token.token_type, expected_type);
+        assert_eq!(token.lexeme, expected_lexeme);
+    }
+    fn assert_tokens(source: String, expected_tokens: &Vec<TokenType>) {
+        let mut scanner = scanner::Scanner::new(&source);
+        for expected in expected_tokens {
+            let actual = scanner.scan_token();
+            assert_eq!(actual.token_type, *expected);
+        }
+
+        assert_eq!(scanner.scan_token().token_type, TokenType::TokenEof);
+    }
+}

src/vm.rs

@@ -1,7 +1,7 @@
-use crate::compiler::compile;
+use crate::chunk::{Chunk, ConversionError, OptCode};
+use crate::compiler;
 use crate::debug::{disassemble_instruction, print_value, trace_enabled};
-use crate::Value;
-use crate::{Chunk, ConversionError, OptCode};
+use crate::value::Value;
 
 pub struct VM<'a> {
     chunk: Option<&'a Chunk>,
@@ -29,9 +29,17 @@ impl<'a> VM<'a> {
         self.ip = 0;
         self.run()
     }
-    pub fn interpret(&mut self, source: &str) -> InterpretResult {
-        compile(source);
-        InterpretResult::InterpretOk
+    pub fn interpret(&mut self, source: &str, chunk: &'a mut Chunk) -> InterpretResult {
+        match compiler::compile(source, &chunk) {
+            false => {
+                return InterpretResult::InterpretCompileError;
+            }
+            true => {
+                self.chunk = Some(chunk);
+                self.ip = 0;
+                return self.run();
+            }
+        }
     }
 
     fn read_byte(&mut self) -> u8 {