diff --git a/examples/arithmetic/main.rs b/examples/arithmetic/main.rs index 046ad8cb..c1a3a7d0 100644 --- a/examples/arithmetic/main.rs +++ b/examples/arithmetic/main.rs @@ -3,6 +3,12 @@ use winnow::prelude::*; mod parser; mod parser_ast; mod parser_lexer; +#[cfg(test)] +mod test_parser; +#[cfg(test)] +mod test_parser_ast; +#[cfg(test)] +mod test_parser_lexer; fn main() -> Result<(), lexopt::Error> { let args = Args::parse()?; @@ -31,9 +37,10 @@ fn calc( println!(" {:#?}={}", result, result.eval()); } Impl::Lexer => { - let tokens = parser_lexer::lex.parse(input)?; + let tokens = parser_lexer::tokens.parse(input)?; println!(" {tokens:#?}"); - let result = parser_lexer::expr.parse(tokens.as_slice()).unwrap(); + let tokens = parser_lexer::Tokens::new(&tokens); + let result = parser_lexer::expr.parse(tokens).unwrap(); println!(" {:#?}={}", result, result.eval()); } } diff --git a/examples/arithmetic/parser.rs b/examples/arithmetic/parser.rs index 45460de1..61a2357d 100644 --- a/examples/arithmetic/parser.rs +++ b/examples/arithmetic/parser.rs @@ -32,7 +32,7 @@ pub(crate) fn expr(i: &mut &str) -> Result { // We read an initial factor and for each time we find // a * or / operator followed by another factor, we do // the math by folding everything -fn term(i: &mut &str) -> Result { +pub(crate) fn term(i: &mut &str) -> Result { let init = factor.parse_next(i)?; repeat(0.., (one_of(['*', '/']), factor)) @@ -53,7 +53,7 @@ fn term(i: &mut &str) -> Result { // We look for a digit suite, and try to convert it. // If either str::from_utf8 or FromStr::from_str fail, // we fallback to the parens parser defined above -fn factor(i: &mut &str) -> Result { +pub(crate) fn factor(i: &mut &str) -> Result { delimited( multispaces, alt((digits.try_map(FromStr::from_str), parens)), @@ -66,71 +66,3 @@ fn factor(i: &mut &str) -> Result { fn parens(i: &mut &str) -> Result { delimited('(', expr, ')').parse_next(i) } - -#[test] -fn factor_test() { - let input = "3"; - let expected = Ok(("", 3)); - assert_eq!(factor.parse_peek(input), expected); - - let input = " 12"; - let expected = Ok(("", 12)); - assert_eq!(factor.parse_peek(input), expected); - - let input = "537 "; - let expected = Ok(("", 537)); - assert_eq!(factor.parse_peek(input), expected); - - let input = " 24 "; - let expected = Ok(("", 24)); - assert_eq!(factor.parse_peek(input), expected); -} - -#[test] -fn term_test() { - let input = " 12 *2 / 3"; - let expected = Ok(("", 8)); - assert_eq!(term.parse_peek(input), expected); - - let input = " 12 *2 / 3"; - let expected = Ok(("", 8)); - assert_eq!(term.parse_peek(input), expected); - - let input = " 2* 3 *2 *2 / 3"; - let expected = Ok(("", 8)); - assert_eq!(term.parse_peek(input), expected); - - let input = " 48 / 3/2"; - let expected = Ok(("", 8)); - assert_eq!(term.parse_peek(input), expected); -} - -#[test] -fn expr_test() { - let input = " 1 + 2 "; - let expected = Ok(("", 3)); - assert_eq!(expr.parse_peek(input), expected); - - let input = " 12 + 6 - 4+ 3"; - let expected = Ok(("", 17)); - assert_eq!(expr.parse_peek(input), expected); - - let input = " 1 + 2*3 + 4"; - let expected = Ok(("", 11)); - assert_eq!(expr.parse_peek(input), expected); -} - -#[test] -fn parens_test() { - let input = " ( 2 )"; - let expected = Ok(("", 2)); - assert_eq!(expr.parse_peek(input), expected); - - let input = " 2* ( 3 + 4 ) "; - let expected = Ok(("", 14)); - assert_eq!(expr.parse_peek(input), expected); - - let input = " 2*2 / ( 5 - 1) + 3"; - let expected = Ok(("", 4)); - assert_eq!(expr.parse_peek(input), expected); -} diff --git a/examples/arithmetic/parser_ast.rs b/examples/arithmetic/parser_ast.rs index 1cd359b0..80887b90 100644 --- a/examples/arithmetic/parser_ast.rs +++ b/examples/arithmetic/parser_ast.rs @@ -67,7 +67,7 @@ pub(crate) fn expr(i: &mut &str) -> Result { .parse_next(i) } -fn term(i: &mut &str) -> Result { +pub(crate) fn term(i: &mut &str) -> Result { let init = factor.parse_next(i)?; repeat(0.., (one_of(['*', '/']), factor)) @@ -84,7 +84,7 @@ fn term(i: &mut &str) -> Result { .parse_next(i) } -fn factor(i: &mut &str) -> Result { +pub(crate) fn factor(i: &mut &str) -> Result { delimited( multispaces, alt((digits.try_map(FromStr::from_str).map(Expr::Value), parens)), @@ -98,86 +98,3 @@ fn parens(i: &mut &str) -> Result { .map(|e| Expr::Paren(Box::new(e))) .parse_next(i) } - -#[test] -fn factor_test() { - let input = "3"; - let expected = Ok(("", String::from("Value(3)"))); - assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 12"; - let expected = Ok(("", String::from("Value(12)"))); - assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = "537 "; - let expected = Ok(("", String::from("Value(537)"))); - assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 24 "; - let expected = Ok(("", String::from("Value(24)"))); - assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); -} - -#[test] -fn term_test() { - let input = " 12 *2 / 3"; - let expected = Ok(("", String::from("Div(Mul(Value(12), Value(2)), Value(3))"))); - assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 12 *2 / 3"; - let expected = Ok(("", String::from("Div(Mul(Value(12), Value(2)), Value(3))"))); - assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 2* 3 *2 *2 / 3"; - let expected = Ok(( - "", - String::from("Div(Mul(Mul(Mul(Value(2), Value(3)), Value(2)), Value(2)), Value(3))"), - )); - assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 48 / 3/2"; - let expected = Ok(("", String::from("Div(Div(Value(48), Value(3)), Value(2))"))); - assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); -} - -#[test] -fn expr_test() { - let input = " 1 + 2 "; - let expected = Ok(("", String::from("Add(Value(1), Value(2))"))); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 12 + 6 - 4+ 3"; - let expected = Ok(( - "", - String::from("Add(Sub(Add(Value(12), Value(6)), Value(4)), Value(3))"), - )); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 1 + 2*3 + 4"; - let expected = Ok(( - "", - String::from("Add(Add(Value(1), Mul(Value(2), Value(3))), Value(4))"), - )); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); -} - -#[test] -fn parens_test() { - let input = " ( 2 )"; - let expected = Ok(("", String::from("Paren(Value(2))"))); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 2* ( 3 + 4 ) "; - let expected = Ok(( - "", - String::from("Mul(Value(2), Paren(Add(Value(3), Value(4))))"), - )); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 2*2 / ( 5 - 1) + 3"; - let expected = Ok(( - "", - String::from("Add(Div(Mul(Value(2), Value(2)), Paren(Sub(Value(5), Value(1)))), Value(3))"), - )); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); -} diff --git a/examples/arithmetic/parser_lexer.rs b/examples/arithmetic/parser_lexer.rs index 8bc2c685..1625ff55 100644 --- a/examples/arithmetic/parser_lexer.rs +++ b/examples/arithmetic/parser_lexer.rs @@ -1,65 +1,60 @@ use std::fmt; use std::fmt::{Debug, Display, Formatter}; -use std::str::FromStr; - use winnow::prelude::*; use winnow::Result; use winnow::{ ascii::{digit1 as digits, multispace0 as multispaces}, combinator::alt, combinator::dispatch, + combinator::eof, combinator::fail, + combinator::opt, combinator::peek, combinator::repeat, combinator::{delimited, preceded, terminated}, + error::ContextError, + stream::TokenSlice, token::any, + token::literal, token::one_of, + token::take_till, }; -#[derive(Debug, Clone)] -pub(crate) enum Expr { - Value(i64), - Add(Box, Box), - Sub(Box, Box), - Mul(Box, Box), - Div(Box, Box), - Paren(Box), +/// Lex and parse +#[allow(dead_code)] +pub(crate) fn expr2(i: &mut &str) -> Result { + let tokens = tokens.parse_next(i)?; + let mut tokens = Tokens::new(&tokens); + expr.parse_next(&mut tokens) } -impl Expr { - pub(crate) fn eval(&self) -> i64 { - match self { - Self::Value(v) => *v, - Self::Add(lhs, rhs) => lhs.eval() + rhs.eval(), - Self::Sub(lhs, rhs) => lhs.eval() - rhs.eval(), - Self::Mul(lhs, rhs) => lhs.eval() * rhs.eval(), - Self::Div(lhs, rhs) => lhs.eval() / rhs.eval(), - Self::Paren(expr) => expr.eval(), - } - } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Token<'s> { + kind: TokenKind, + raw: &'s str, } -impl Display for Expr { - fn fmt(&self, format: &mut Formatter<'_>) -> fmt::Result { - use Expr::{Add, Div, Mul, Paren, Sub, Value}; - match *self { - Value(val) => write!(format, "{val}"), - Add(ref left, ref right) => write!(format, "{left} + {right}"), - Sub(ref left, ref right) => write!(format, "{left} - {right}"), - Mul(ref left, ref right) => write!(format, "{left} * {right}"), - Div(ref left, ref right) => write!(format, "{left} / {right}"), - Paren(ref expr) => write!(format, "({expr})"), - } +impl PartialEq for Token<'_> { + fn eq(&self, other: &TokenKind) -> bool { + self.kind == *other } } #[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub enum Token { - Value(i64), +pub enum TokenKind { + Value, Oper(Oper), OpenParen, CloseParen, + Unknown, + Eof, +} + +impl<'i> Parser, &'i Token<'i>, ContextError> for TokenKind { + fn parse_next(&mut self, input: &mut Tokens<'i>) -> Result<&'i Token<'i>> { + literal(*self).parse_next(input).map(|t| &t[0]) + } } #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -70,95 +65,149 @@ pub enum Oper { Div, } -impl winnow::stream::ContainsToken for Token { +impl winnow::stream::ContainsToken<&'_ Token<'_>> for TokenKind { #[inline(always)] - fn contains_token(&self, token: Token) -> bool { - *self == token + fn contains_token(&self, token: &'_ Token<'_>) -> bool { + *self == token.kind } } -impl winnow::stream::ContainsToken for &'_ [Token] { +impl winnow::stream::ContainsToken<&'_ Token<'_>> for &'_ [TokenKind] { #[inline] - fn contains_token(&self, token: Token) -> bool { - self.iter().any(|t| *t == token) + fn contains_token(&self, token: &'_ Token<'_>) -> bool { + self.iter().any(|t| *t == token.kind) } } -impl winnow::stream::ContainsToken for &'_ [Token; LEN] { +impl winnow::stream::ContainsToken<&'_ Token<'_>> for &'_ [TokenKind; LEN] { #[inline] - fn contains_token(&self, token: Token) -> bool { - self.iter().any(|t| *t == token) + fn contains_token(&self, token: &'_ Token<'_>) -> bool { + self.iter().any(|t| *t == token.kind) } } -impl winnow::stream::ContainsToken for [Token; LEN] { +impl winnow::stream::ContainsToken<&'_ Token<'_>> for [TokenKind; LEN] { #[inline] - fn contains_token(&self, token: Token) -> bool { - self.iter().any(|t| *t == token) + fn contains_token(&self, token: &'_ Token<'_>) -> bool { + self.iter().any(|t| *t == token.kind) } } -#[allow(dead_code)] -pub(crate) fn expr2(i: &mut &str) -> Result { - let tokens = lex.parse_next(i)?; - expr.parse_next(&mut tokens.as_slice()) -} - -pub(crate) fn lex(i: &mut &str) -> Result> { - preceded(multispaces, repeat(1.., terminated(token, multispaces))).parse_next(i) +/// Lex tokens +/// +/// See [`expr`] to parse the tokens +pub(crate) fn tokens<'s>(i: &mut &'s str) -> Result>> { + let mut tokens: Vec<_> = + preceded(multispaces, repeat(1.., terminated(token, multispaces))).parse_next(i)?; + if let Some(eof) = opt(eof.map(|raw| Token { + kind: TokenKind::Eof, + raw, + })) + .parse_next(i)? + { + tokens.push(eof); + } + Ok(tokens) } -fn token(i: &mut &str) -> Result { +fn token<'s>(i: &mut &'s str) -> Result> { dispatch! {peek(any); - '0'..='9' => digits.try_map(FromStr::from_str).map(Token::Value), - '(' => '('.value(Token::OpenParen), - ')' => ')'.value(Token::CloseParen), - '+' => '+'.value(Token::Oper(Oper::Add)), - '-' => '-'.value(Token::Oper(Oper::Sub)), - '*' => '*'.value(Token::Oper(Oper::Mul)), - '/' => '/'.value(Token::Oper(Oper::Div)), - _ => fail, + '0'..='9' => digits.value(TokenKind::Value), + '(' => '('.value(TokenKind::OpenParen), + ')' => ')'.value(TokenKind::CloseParen), + '+' => '+'.value(TokenKind::Oper(Oper::Add)), + '-' => '-'.value(TokenKind::Oper(Oper::Sub)), + '*' => '*'.value(TokenKind::Oper(Oper::Mul)), + '/' => '/'.value(TokenKind::Oper(Oper::Div)), + ' '| '\t'| '\r'| '\n' => fail, + _ => take_till(.., ('0'..='9', '(', ')', '+', '-', '*', '/')).value(TokenKind::Unknown), } + .with_taken() + .map(|(kind, raw)| Token { kind, raw }) .parse_next(i) } -pub(crate) fn expr(i: &mut &[Token]) -> Result { +#[derive(Debug, Clone)] +pub(crate) enum Expr { + Value(i64), + Add(Box, Box), + Sub(Box, Box), + Mul(Box, Box), + Div(Box, Box), + Paren(Box), +} + +impl Expr { + pub(crate) fn eval(&self) -> i64 { + match self { + Self::Value(v) => *v, + Self::Add(lhs, rhs) => lhs.eval() + rhs.eval(), + Self::Sub(lhs, rhs) => lhs.eval() - rhs.eval(), + Self::Mul(lhs, rhs) => lhs.eval() * rhs.eval(), + Self::Div(lhs, rhs) => lhs.eval() / rhs.eval(), + Self::Paren(expr) => expr.eval(), + } + } +} + +impl Display for Expr { + fn fmt(&self, format: &mut Formatter<'_>) -> fmt::Result { + use Expr::{Add, Div, Mul, Paren, Sub, Value}; + match *self { + Value(val) => write!(format, "{val}"), + Add(ref left, ref right) => write!(format, "{left} + {right}"), + Sub(ref left, ref right) => write!(format, "{left} - {right}"), + Mul(ref left, ref right) => write!(format, "{left} * {right}"), + Div(ref left, ref right) => write!(format, "{left} / {right}"), + Paren(ref expr) => write!(format, "({expr})"), + } + } +} + +pub(crate) type Tokens<'i> = TokenSlice<'i, Token<'i>>; + +/// Parse the tokens lexed in [`tokens`] +pub(crate) fn expr(i: &mut Tokens<'_>) -> Result { let init = term.parse_next(i)?; - repeat( + let expr = repeat( 0.., ( - one_of([Token::Oper(Oper::Add), Token::Oper(Oper::Sub)]), + one_of([TokenKind::Oper(Oper::Add), TokenKind::Oper(Oper::Sub)]), term, ), ) .fold( move || init.clone(), - |acc, (op, val): (Token, Expr)| { - if op == Token::Oper(Oper::Add) { + |acc, (op, val): (&Token<'_>, Expr)| { + if op.kind == TokenKind::Oper(Oper::Add) { Expr::Add(Box::new(acc), Box::new(val)) } else { Expr::Sub(Box::new(acc), Box::new(val)) } }, ) - .parse_next(i) + .parse_next(i)?; + + opt(TokenKind::Eof).parse_next(i)?; + + Ok(expr) } -fn term(i: &mut &[Token]) -> Result { +pub(crate) fn term(i: &mut Tokens<'_>) -> Result { let init = factor.parse_next(i)?; repeat( 0.., ( - one_of([Token::Oper(Oper::Mul), Token::Oper(Oper::Div)]), + one_of([TokenKind::Oper(Oper::Mul), TokenKind::Oper(Oper::Div)]), factor, ), ) .fold( move || init.clone(), - |acc, (op, val): (Token, Expr)| { - if op == Token::Oper(Oper::Mul) { + |acc, (op, val): (&Token<'_>, Expr)| { + if op.kind == TokenKind::Oper(Oper::Mul) { Expr::Mul(Box::new(acc), Box::new(val)) } else { Expr::Div(Box::new(acc), Box::new(val)) @@ -168,134 +217,16 @@ fn term(i: &mut &[Token]) -> Result { .parse_next(i) } -fn factor(i: &mut &[Token]) -> Result { +pub(crate) fn factor(i: &mut Tokens<'_>) -> Result { alt(( - one_of(|t| matches!(t, Token::Value(_))).map(|t| match t { - Token::Value(v) => Expr::Value(v), - _ => unreachable!(), - }), + TokenKind::Value.try_map(|t: &Token<'_>| t.raw.parse::().map(Expr::Value)), parens, )) .parse_next(i) } -fn parens(i: &mut &[Token]) -> Result { - delimited(one_of(Token::OpenParen), expr, one_of(Token::CloseParen)) +fn parens(i: &mut Tokens<'_>) -> Result { + delimited(TokenKind::OpenParen, expr, TokenKind::CloseParen) .map(|e| Expr::Paren(Box::new(e))) .parse_next(i) } - -#[test] -fn lex_test() { - let input = "3"; - let expected = Ok(String::from(r#"("", [Value(3)])"#)); - assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); - - let input = " 24 "; - let expected = Ok(String::from(r#"("", [Value(24)])"#)); - assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); - - let input = " 12 *2 / 3"; - let expected = Ok(String::from( - r#"("", [Value(12), Oper(Mul), Value(2), Oper(Div), Value(3)])"#, - )); - assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); - - let input = " 2*2 / ( 5 - 1) + 3"; - let expected = Ok(String::from( - r#"("", [Value(2), Oper(Mul), Value(2), Oper(Div), OpenParen, Value(5), Oper(Sub), Value(1), CloseParen, Oper(Add), Value(3)])"#, - )); - assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); -} - -#[test] -fn factor_test() { - let input = "3"; - let expected = Ok(String::from("Value(3)")); - let input = lex.parse(input).unwrap(); - assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 12"; - let expected = Ok(String::from("Value(12)")); - let input = lex.parse(input).unwrap(); - assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = "537 "; - let expected = Ok(String::from("Value(537)")); - let input = lex.parse(input).unwrap(); - assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 24 "; - let expected = Ok(String::from("Value(24)")); - let input = lex.parse(input).unwrap(); - assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); -} - -#[test] -fn term_test() { - let input = " 12 *2 / 3"; - let expected = Ok(String::from("Div(Mul(Value(12), Value(2)), Value(3))")); - let input = lex.parse(input).unwrap(); - assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 12 *2 / 3"; - let expected = Ok(String::from("Div(Mul(Value(12), Value(2)), Value(3))")); - let input = lex.parse(input).unwrap(); - assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 2* 3 *2 *2 / 3"; - let expected = Ok(String::from( - "Div(Mul(Mul(Mul(Value(2), Value(3)), Value(2)), Value(2)), Value(3))", - )); - let input = lex.parse(input).unwrap(); - assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 48 / 3/2"; - let expected = Ok(String::from("Div(Div(Value(48), Value(3)), Value(2))")); - let input = lex.parse(input).unwrap(); - assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); -} - -#[test] -fn expr_test() { - let input = " 1 + 2 "; - let expected = Ok(String::from("Add(Value(1), Value(2))")); - let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 12 + 6 - 4+ 3"; - let expected = Ok(String::from( - "Add(Sub(Add(Value(12), Value(6)), Value(4)), Value(3))", - )); - let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 1 + 2*3 + 4"; - let expected = Ok(String::from( - "Add(Add(Value(1), Mul(Value(2), Value(3))), Value(4))", - )); - let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); -} - -#[test] -fn parens_test() { - let input = " ( 2 )"; - let expected = Ok(String::from("Paren(Value(2))")); - let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 2* ( 3 + 4 ) "; - let expected = Ok(String::from( - "Mul(Value(2), Paren(Add(Value(3), Value(4))))", - )); - let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 2*2 / ( 5 - 1) + 3"; - let expected = Ok(String::from( - "Add(Div(Mul(Value(2), Value(2)), Paren(Sub(Value(5), Value(1)))), Value(3))", - )); - let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); -} diff --git a/examples/arithmetic/test_parser.rs b/examples/arithmetic/test_parser.rs new file mode 100644 index 00000000..f22ba62b --- /dev/null +++ b/examples/arithmetic/test_parser.rs @@ -0,0 +1,186 @@ +use snapbox::assert_data_eq; +use snapbox::prelude::*; +use snapbox::str; +use winnow::prelude::*; + +use crate::parser::*; + +#[test] +fn factor_test() { + let input = "3"; + let expected = str![[r#" +Ok( + ( + "", + 3, + ), +) + +"#]]; + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); + + let input = " 12"; + let expected = str![[r#" +Ok( + ( + "", + 12, + ), +) + +"#]]; + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); + + let input = "537 "; + let expected = str![[r#" +Ok( + ( + "", + 537, + ), +) + +"#]]; + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); + + let input = " 24 "; + let expected = str![[r#" +Ok( + ( + "", + 24, + ), +) + +"#]]; + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); +} + +#[test] +fn term_test() { + let input = " 12 *2 / 3"; + let expected = str![[r#" +Ok( + ( + "", + 8, + ), +) + +"#]]; + assert_data_eq!(term.parse_peek(input).to_debug(), expected); + + let input = " 12 *2 / 3"; + let expected = str![[r#" +Ok( + ( + "", + 8, + ), +) + +"#]]; + assert_data_eq!(term.parse_peek(input).to_debug(), expected); + + let input = " 2* 3 *2 *2 / 3"; + let expected = str![[r#" +Ok( + ( + "", + 8, + ), +) + +"#]]; + assert_data_eq!(term.parse_peek(input).to_debug(), expected); + + let input = " 48 / 3/2"; + let expected = str![[r#" +Ok( + ( + "", + 8, + ), +) + +"#]]; + assert_data_eq!(term.parse_peek(input).to_debug(), expected); +} + +#[test] +fn expr_test() { + let input = " 1 + 2 "; + let expected = str![[r#" +Ok( + ( + "", + 3, + ), +) + +"#]]; + assert_data_eq!(expr.parse_peek(input).to_debug(), expected); + + let input = " 12 + 6 - 4+ 3"; + let expected = str![[r#" +Ok( + ( + "", + 17, + ), +) + +"#]]; + assert_data_eq!(expr.parse_peek(input).to_debug(), expected); + + let input = " 1 + 2*3 + 4"; + let expected = str![[r#" +Ok( + ( + "", + 11, + ), +) + +"#]]; + assert_data_eq!(expr.parse_peek(input).to_debug(), expected); +} + +#[test] +fn parens_test() { + let input = " ( 2 )"; + let expected = str![[r#" +Ok( + ( + "", + 2, + ), +) + +"#]]; + assert_data_eq!(expr.parse_peek(input).to_debug(), expected); + + let input = " 2* ( 3 + 4 ) "; + let expected = str![[r#" +Ok( + ( + "", + 14, + ), +) + +"#]]; + assert_data_eq!(expr.parse_peek(input).to_debug(), expected); + + let input = " 2*2 / ( 5 - 1) + 3"; + let expected = str![[r#" +Ok( + ( + "", + 4, + ), +) + +"#]]; + assert_data_eq!(expr.parse_peek(input).to_debug(), expected); +} diff --git a/examples/arithmetic/test_parser_ast.rs b/examples/arithmetic/test_parser_ast.rs new file mode 100644 index 00000000..86c08be7 --- /dev/null +++ b/examples/arithmetic/test_parser_ast.rs @@ -0,0 +1,317 @@ +use snapbox::assert_data_eq; +use snapbox::prelude::*; +use snapbox::str; +use winnow::prelude::*; + +use crate::parser_ast::*; + +#[test] +fn factor_test() { + let input = "3"; + let expected = str![[r#" +Ok( + ( + "", + Value( + 3, + ), + ), +) + +"#]]; + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); + + let input = " 12"; + let expected = str![[r#" +Ok( + ( + "", + Value( + 12, + ), + ), +) + +"#]]; + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); + + let input = "537 "; + let expected = str![[r#" +Ok( + ( + "", + Value( + 537, + ), + ), +) + +"#]]; + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); + + let input = " 24 "; + let expected = str![[r#" +Ok( + ( + "", + Value( + 24, + ), + ), +) + +"#]]; + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); +} + +#[test] +fn term_test() { + let input = " 12 *2 / 3"; + let expected = str![[r#" +Ok( + ( + "", + Div( + Mul( + Value( + 12, + ), + Value( + 2, + ), + ), + Value( + 3, + ), + ), + ), +) + +"#]]; + assert_data_eq!(term.parse_peek(input).to_debug(), expected); + + let input = " 12 *2 / 3"; + let expected = str![[r#" +Ok( + ( + "", + Div( + Mul( + Value( + 12, + ), + Value( + 2, + ), + ), + Value( + 3, + ), + ), + ), +) + +"#]]; + assert_data_eq!(term.parse_peek(input).to_debug(), expected); + + let input = " 2* 3 *2 *2 / 3"; + let expected = str![[r#" +Ok( + ( + "", + Div( + Mul( + Mul( + Mul( + Value( + 2, + ), + Value( + 3, + ), + ), + Value( + 2, + ), + ), + Value( + 2, + ), + ), + Value( + 3, + ), + ), + ), +) + +"#]]; + assert_data_eq!(term.parse_peek(input).to_debug(), expected); + + let input = " 48 / 3/2"; + let expected = str![[r#" +Ok( + ( + "", + Div( + Div( + Value( + 48, + ), + Value( + 3, + ), + ), + Value( + 2, + ), + ), + ), +) + +"#]]; + assert_data_eq!(term.parse_peek(input).to_debug(), expected); +} + +#[test] +fn expr_test() { + let input = " 1 + 2 "; + let expected = str![[r#" +Ok( + Add( + Value( + 1, + ), + Value( + 2, + ), + ), +) + +"#]]; + assert_data_eq!(expr.parse(input).to_debug(), expected); + + let input = " 12 + 6 - 4+ 3"; + let expected = str![[r#" +Ok( + Add( + Sub( + Add( + Value( + 12, + ), + Value( + 6, + ), + ), + Value( + 4, + ), + ), + Value( + 3, + ), + ), +) + +"#]]; + assert_data_eq!(expr.parse(input).to_debug(), expected); + + let input = " 1 + 2*3 + 4"; + let expected = str![[r#" +Ok( + Add( + Add( + Value( + 1, + ), + Mul( + Value( + 2, + ), + Value( + 3, + ), + ), + ), + Value( + 4, + ), + ), +) + +"#]]; + assert_data_eq!(expr.parse(input).to_debug(), expected); +} + +#[test] +fn parens_test() { + let input = " ( 2 )"; + let expected = str![[r#" +Ok( + Paren( + Value( + 2, + ), + ), +) + +"#]]; + assert_data_eq!(expr.parse(input).to_debug(), expected); + + let input = " 2* ( 3 + 4 ) "; + let expected = str![[r#" +Ok( + Mul( + Value( + 2, + ), + Paren( + Add( + Value( + 3, + ), + Value( + 4, + ), + ), + ), + ), +) + +"#]]; + assert_data_eq!(expr.parse(input).to_debug(), expected); + + let input = " 2*2 / ( 5 - 1) + 3"; + let expected = str![[r#" +Ok( + Add( + Div( + Mul( + Value( + 2, + ), + Value( + 2, + ), + ), + Paren( + Sub( + Value( + 5, + ), + Value( + 1, + ), + ), + ), + ), + Value( + 3, + ), + ), +) + +"#]]; + assert_data_eq!(expr.parse(input).to_debug(), expected); +} diff --git a/examples/arithmetic/test_parser_lexer.rs b/examples/arithmetic/test_parser_lexer.rs new file mode 100644 index 00000000..af08487e --- /dev/null +++ b/examples/arithmetic/test_parser_lexer.rs @@ -0,0 +1,736 @@ +use snapbox::assert_data_eq; +use snapbox::prelude::*; +use snapbox::str; +use winnow::prelude::*; + +use crate::parser_lexer::*; + +#[test] +fn lex_test() { + let input = "3"; + let expected = str![[r#" +Ok( + ( + "", + [ + Token { + kind: Value, + raw: "3", + }, + Token { + kind: Eof, + raw: "", + }, + ], + ), +) + +"#]]; + assert_data_eq!(tokens.parse_peek(input).to_debug(), expected); + + let input = " 24 "; + let expected = str![[r#" +Ok( + ( + "", + [ + Token { + kind: Value, + raw: "24", + }, + Token { + kind: Eof, + raw: "", + }, + ], + ), +) + +"#]]; + assert_data_eq!(tokens.parse_peek(input).to_debug(), expected); + + let input = " 12 *2 / 3"; + let expected = str![[r#" +Ok( + ( + "", + [ + Token { + kind: Value, + raw: "12", + }, + Token { + kind: Oper( + Mul, + ), + raw: "*", + }, + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Div, + ), + raw: "/", + }, + Token { + kind: Value, + raw: "3", + }, + Token { + kind: Eof, + raw: "", + }, + ], + ), +) + +"#]]; + assert_data_eq!(tokens.parse_peek(input).to_debug(), expected); + + let input = " 2*2 / ( 5 - 1) + 3"; + let expected = str![[r#" +Ok( + ( + "", + [ + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Mul, + ), + raw: "*", + }, + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Div, + ), + raw: "/", + }, + Token { + kind: OpenParen, + raw: "(", + }, + Token { + kind: Value, + raw: "5", + }, + Token { + kind: Oper( + Sub, + ), + raw: "-", + }, + Token { + kind: Value, + raw: "1", + }, + Token { + kind: CloseParen, + raw: ")", + }, + Token { + kind: Oper( + Add, + ), + raw: "+", + }, + Token { + kind: Value, + raw: "3", + }, + Token { + kind: Eof, + raw: "", + }, + ], + ), +) + +"#]]; + assert_data_eq!(tokens.parse_peek(input).to_debug(), expected); +} + +#[test] +fn factor_test() { + let input = "3"; + let expected = str![[r#" +Ok( + ( + TokenSlice { + initial: [ + Token { + kind: Value, + raw: "3", + }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, + ], + }, + Value( + 3, + ), + ), +) + +"#]]; + let input = tokens.parse(input).unwrap(); + let input = Tokens::new(&input); + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); + + let input = " 12"; + let expected = str![[r#" +Ok( + ( + TokenSlice { + initial: [ + Token { + kind: Value, + raw: "12", + }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, + ], + }, + Value( + 12, + ), + ), +) + +"#]]; + let input = tokens.parse(input).unwrap(); + let input = Tokens::new(&input); + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); + + let input = "537 "; + let expected = str![[r#" +Ok( + ( + TokenSlice { + initial: [ + Token { + kind: Value, + raw: "537", + }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, + ], + }, + Value( + 537, + ), + ), +) + +"#]]; + let input = tokens.parse(input).unwrap(); + let input = Tokens::new(&input); + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); + + let input = " 24 "; + let expected = str![[r#" +Ok( + ( + TokenSlice { + initial: [ + Token { + kind: Value, + raw: "24", + }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, + ], + }, + Value( + 24, + ), + ), +) + +"#]]; + let input = tokens.parse(input).unwrap(); + let input = Tokens::new(&input); + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); +} + +#[test] +fn term_test() { + let input = " 12 *2 / 3"; + let expected = str![[r#" +Ok( + ( + TokenSlice { + initial: [ + Token { + kind: Value, + raw: "12", + }, + Token { + kind: Oper( + Mul, + ), + raw: "*", + }, + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Div, + ), + raw: "/", + }, + Token { + kind: Value, + raw: "3", + }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, + ], + }, + Div( + Mul( + Value( + 12, + ), + Value( + 2, + ), + ), + Value( + 3, + ), + ), + ), +) + +"#]]; + let input = tokens.parse(input).unwrap(); + let input = Tokens::new(&input); + assert_data_eq!(term.parse_peek(input).to_debug(), expected); + + let input = " 12 *2 / 3"; + let expected = str![[r#" +Ok( + ( + TokenSlice { + initial: [ + Token { + kind: Value, + raw: "12", + }, + Token { + kind: Oper( + Mul, + ), + raw: "*", + }, + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Div, + ), + raw: "/", + }, + Token { + kind: Value, + raw: "3", + }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, + ], + }, + Div( + Mul( + Value( + 12, + ), + Value( + 2, + ), + ), + Value( + 3, + ), + ), + ), +) + +"#]]; + let input = tokens.parse(input).unwrap(); + let input = Tokens::new(&input); + assert_data_eq!(term.parse_peek(input).to_debug(), expected); + + let input = " 2* 3 *2 *2 / 3"; + let expected = str![[r#" +Ok( + ( + TokenSlice { + initial: [ + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Mul, + ), + raw: "*", + }, + Token { + kind: Value, + raw: "3", + }, + Token { + kind: Oper( + Mul, + ), + raw: "*", + }, + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Mul, + ), + raw: "*", + }, + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Div, + ), + raw: "/", + }, + Token { + kind: Value, + raw: "3", + }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, + ], + }, + Div( + Mul( + Mul( + Mul( + Value( + 2, + ), + Value( + 3, + ), + ), + Value( + 2, + ), + ), + Value( + 2, + ), + ), + Value( + 3, + ), + ), + ), +) + +"#]]; + let input = tokens.parse(input).unwrap(); + let input = Tokens::new(&input); + assert_data_eq!(term.parse_peek(input).to_debug(), expected); + + let input = " 48 / 3/2"; + let expected = str![[r#" +Ok( + ( + TokenSlice { + initial: [ + Token { + kind: Value, + raw: "48", + }, + Token { + kind: Oper( + Div, + ), + raw: "/", + }, + Token { + kind: Value, + raw: "3", + }, + Token { + kind: Oper( + Div, + ), + raw: "/", + }, + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, + ], + }, + Div( + Div( + Value( + 48, + ), + Value( + 3, + ), + ), + Value( + 2, + ), + ), + ), +) + +"#]]; + let input = tokens.parse(input).unwrap(); + let input = Tokens::new(&input); + assert_data_eq!(term.parse_peek(input).to_debug(), expected); +} + +#[test] +fn expr_test() { + let input = " 1 + 2 "; + let expected = str![[r#" +Ok( + Add( + Value( + 1, + ), + Value( + 2, + ), + ), +) + +"#]]; + let input = tokens.parse(input).unwrap(); + let input = Tokens::new(&input); + assert_data_eq!(expr.parse(input).to_debug(), expected); + + let input = " 12 + 6 - 4+ 3"; + let expected = str![[r#" +Ok( + Add( + Sub( + Add( + Value( + 12, + ), + Value( + 6, + ), + ), + Value( + 4, + ), + ), + Value( + 3, + ), + ), +) + +"#]]; + let input = tokens.parse(input).unwrap(); + let input = Tokens::new(&input); + assert_data_eq!(expr.parse(input).to_debug(), expected); + + let input = " 1 + 2*3 + 4"; + let expected = str![[r#" +Ok( + Add( + Add( + Value( + 1, + ), + Mul( + Value( + 2, + ), + Value( + 3, + ), + ), + ), + Value( + 4, + ), + ), +) + +"#]]; + let input = tokens.parse(input).unwrap(); + let input = Tokens::new(&input); + assert_data_eq!(expr.parse(input).to_debug(), expected); +} + +#[test] +fn parens_test() { + let input = " ( 2 )"; + let expected = str![[r#" +Ok( + Paren( + Value( + 2, + ), + ), +) + +"#]]; + let input = tokens.parse(input).unwrap(); + let input = Tokens::new(&input); + assert_data_eq!(expr.parse(input).to_debug(), expected); + + let input = " 2* ( 3 + 4 ) "; + let expected = str![[r#" +Ok( + Mul( + Value( + 2, + ), + Paren( + Add( + Value( + 3, + ), + Value( + 4, + ), + ), + ), + ), +) + +"#]]; + let input = tokens.parse(input).unwrap(); + let input = Tokens::new(&input); + assert_data_eq!(expr.parse(input).to_debug(), expected); + + let input = " 2*2 / ( 5 - 1) + 3"; + let expected = str![[r#" +Ok( + Add( + Div( + Mul( + Value( + 2, + ), + Value( + 2, + ), + ), + Paren( + Sub( + Value( + 5, + ), + Value( + 1, + ), + ), + ), + ), + Value( + 3, + ), + ), +) + +"#]]; + let input = tokens.parse(input).unwrap(); + let input = Tokens::new(&input); + assert_data_eq!(expr.parse(input).to_debug(), expected); +} diff --git a/src/_topic/arithmetic.rs b/src/_topic/arithmetic.rs index d94b4fa4..dce37111 100644 --- a/src/_topic/arithmetic.rs +++ b/src/_topic/arithmetic.rs @@ -1,19 +1,6 @@ //! # Arithmetic //! -//! ## Direct evaluation -//! +//! This parses arithmetic expressions and directly evaluates them. //! ```rust #![doc = include_str!("../../examples/arithmetic/parser.rs")] //! ``` -//! -//! ## Parse to AST -//! -//! ```rust -#![doc = include_str!("../../examples/arithmetic/parser_ast.rs")] -//! ``` -//! -//! ## Parse to Tokens then AST -//! -//! ```rust -#![doc = include_str!("../../examples/arithmetic/parser_lexer.rs")] -//! ``` diff --git a/src/_topic/lexing.rs b/src/_topic/lexing.rs new file mode 100644 index 00000000..938453ae --- /dev/null +++ b/src/_topic/lexing.rs @@ -0,0 +1,23 @@ +//! # Lexing and Parsing +//! +//! ## Parse to AST +//! +//! The simplest way to write a parser is to parse directly to the AST. +//! +//! Example: +//! ```rust +#![doc = include_str!("../../examples/arithmetic/parser_ast.rs")] +//! ``` +//! +//! ## Lexing +//! +//! However, there are times when you may want to separate lexing from parsing. +//! Winnow provides [`TokenSlice`] to simplify this. +//! +//! Example: +//! ```rust +#![doc = include_str!("../../examples/arithmetic/parser_lexer.rs")] +//! ``` + +#![allow(unused_imports)] +use crate::stream::TokenSlice; diff --git a/src/_topic/mod.rs b/src/_topic/mod.rs index f279ac69..fcf83e0f 100644 --- a/src/_topic/mod.rs +++ b/src/_topic/mod.rs @@ -15,6 +15,7 @@ //! - [Implementing `FromStr`][fromstr] //! - [Performance][performance] //! - [Parsing Partial Input][partial] +//! - [Lexing and Parsing][lexing] //! - [Custom stream or token][stream] //! - [Custom errors][error] //! - [Debugging][crate::_tutorial::chapter_8] @@ -32,6 +33,7 @@ pub mod http; pub mod ini; pub mod json; pub mod language; +pub mod lexing; pub mod nom; pub mod partial; pub mod performance; diff --git a/src/stream/token.rs b/src/stream/token.rs index e4f121b6..97ff2e5f 100644 --- a/src/stream/token.rs +++ b/src/stream/token.rs @@ -15,6 +15,13 @@ use crate::stream::StreamIsPartial; use crate::stream::UpdateSlice; /// Specialized input for parsing lexed tokens +/// +/// Helpful impls +/// - Any `PartialEq` type (e.g. a `TokenKind` or `&str`) can be used with +/// [`literal`][crate::token::literal] +/// - A `PartialEq` for `&str` allows for using `&str` as a parser for tokens +/// +/// See also [Lexing and Parsing][crate::_topic::lexing]. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct TokenSlice<'t, T> { initial: &'t [T],