From fa1f22f56596d75b623b786f627ba3af0a4204aa Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 28 Feb 2025 10:53:02 -0600 Subject: [PATCH 01/11] docs(examples): Pull tests out of arithmetic examples --- examples/arithmetic/main.rs | 6 ++ examples/arithmetic/parser.rs | 72 +------------- examples/arithmetic/parser_ast.rs | 87 +---------------- examples/arithmetic/parser_lexer.rs | 119 +---------------------- examples/arithmetic/test_parser.rs | 71 ++++++++++++++ examples/arithmetic/test_parser_ast.rs | 86 ++++++++++++++++ examples/arithmetic/test_parser_lexer.rs | 118 ++++++++++++++++++++++ 7 files changed, 287 insertions(+), 272 deletions(-) create mode 100644 examples/arithmetic/test_parser.rs create mode 100644 examples/arithmetic/test_parser_ast.rs create mode 100644 examples/arithmetic/test_parser_lexer.rs diff --git a/examples/arithmetic/main.rs b/examples/arithmetic/main.rs index 046ad8cb..d262bf19 100644 --- a/examples/arithmetic/main.rs +++ b/examples/arithmetic/main.rs @@ -3,6 +3,12 @@ use winnow::prelude::*; mod parser; mod parser_ast; mod parser_lexer; +#[cfg(test)] +mod test_parser; +#[cfg(test)] +mod test_parser_ast; +#[cfg(test)] +mod test_parser_lexer; fn main() -> Result<(), lexopt::Error> { let args = Args::parse()?; diff --git a/examples/arithmetic/parser.rs b/examples/arithmetic/parser.rs index 45460de1..61a2357d 100644 --- a/examples/arithmetic/parser.rs +++ b/examples/arithmetic/parser.rs @@ -32,7 +32,7 @@ pub(crate) fn expr(i: &mut &str) -> Result { // We read an initial factor and for each time we find // a * or / operator followed by another factor, we do // the math by folding everything -fn term(i: &mut &str) -> Result { +pub(crate) fn term(i: &mut &str) -> Result { let init = factor.parse_next(i)?; repeat(0.., (one_of(['*', '/']), factor)) @@ -53,7 +53,7 @@ fn term(i: &mut &str) -> Result { // We look for a digit suite, and try to convert it. // If either str::from_utf8 or FromStr::from_str fail, // we fallback to the parens parser defined above -fn factor(i: &mut &str) -> Result { +pub(crate) fn factor(i: &mut &str) -> Result { delimited( multispaces, alt((digits.try_map(FromStr::from_str), parens)), @@ -66,71 +66,3 @@ fn factor(i: &mut &str) -> Result { fn parens(i: &mut &str) -> Result { delimited('(', expr, ')').parse_next(i) } - -#[test] -fn factor_test() { - let input = "3"; - let expected = Ok(("", 3)); - assert_eq!(factor.parse_peek(input), expected); - - let input = " 12"; - let expected = Ok(("", 12)); - assert_eq!(factor.parse_peek(input), expected); - - let input = "537 "; - let expected = Ok(("", 537)); - assert_eq!(factor.parse_peek(input), expected); - - let input = " 24 "; - let expected = Ok(("", 24)); - assert_eq!(factor.parse_peek(input), expected); -} - -#[test] -fn term_test() { - let input = " 12 *2 / 3"; - let expected = Ok(("", 8)); - assert_eq!(term.parse_peek(input), expected); - - let input = " 12 *2 / 3"; - let expected = Ok(("", 8)); - assert_eq!(term.parse_peek(input), expected); - - let input = " 2* 3 *2 *2 / 3"; - let expected = Ok(("", 8)); - assert_eq!(term.parse_peek(input), expected); - - let input = " 48 / 3/2"; - let expected = Ok(("", 8)); - assert_eq!(term.parse_peek(input), expected); -} - -#[test] -fn expr_test() { - let input = " 1 + 2 "; - let expected = Ok(("", 3)); - assert_eq!(expr.parse_peek(input), expected); - - let input = " 12 + 6 - 4+ 3"; - let expected = Ok(("", 17)); - assert_eq!(expr.parse_peek(input), expected); - - let input = " 1 + 2*3 + 4"; - let expected = Ok(("", 11)); - assert_eq!(expr.parse_peek(input), expected); -} - -#[test] -fn parens_test() { - let input = " ( 2 )"; - let expected = Ok(("", 2)); - assert_eq!(expr.parse_peek(input), expected); - - let input = " 2* ( 3 + 4 ) "; - let expected = Ok(("", 14)); - assert_eq!(expr.parse_peek(input), expected); - - let input = " 2*2 / ( 5 - 1) + 3"; - let expected = Ok(("", 4)); - assert_eq!(expr.parse_peek(input), expected); -} diff --git a/examples/arithmetic/parser_ast.rs b/examples/arithmetic/parser_ast.rs index 1cd359b0..80887b90 100644 --- a/examples/arithmetic/parser_ast.rs +++ b/examples/arithmetic/parser_ast.rs @@ -67,7 +67,7 @@ pub(crate) fn expr(i: &mut &str) -> Result { .parse_next(i) } -fn term(i: &mut &str) -> Result { +pub(crate) fn term(i: &mut &str) -> Result { let init = factor.parse_next(i)?; repeat(0.., (one_of(['*', '/']), factor)) @@ -84,7 +84,7 @@ fn term(i: &mut &str) -> Result { .parse_next(i) } -fn factor(i: &mut &str) -> Result { +pub(crate) fn factor(i: &mut &str) -> Result { delimited( multispaces, alt((digits.try_map(FromStr::from_str).map(Expr::Value), parens)), @@ -98,86 +98,3 @@ fn parens(i: &mut &str) -> Result { .map(|e| Expr::Paren(Box::new(e))) .parse_next(i) } - -#[test] -fn factor_test() { - let input = "3"; - let expected = Ok(("", String::from("Value(3)"))); - assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 12"; - let expected = Ok(("", String::from("Value(12)"))); - assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = "537 "; - let expected = Ok(("", String::from("Value(537)"))); - assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 24 "; - let expected = Ok(("", String::from("Value(24)"))); - assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); -} - -#[test] -fn term_test() { - let input = " 12 *2 / 3"; - let expected = Ok(("", String::from("Div(Mul(Value(12), Value(2)), Value(3))"))); - assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 12 *2 / 3"; - let expected = Ok(("", String::from("Div(Mul(Value(12), Value(2)), Value(3))"))); - assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 2* 3 *2 *2 / 3"; - let expected = Ok(( - "", - String::from("Div(Mul(Mul(Mul(Value(2), Value(3)), Value(2)), Value(2)), Value(3))"), - )); - assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 48 / 3/2"; - let expected = Ok(("", String::from("Div(Div(Value(48), Value(3)), Value(2))"))); - assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); -} - -#[test] -fn expr_test() { - let input = " 1 + 2 "; - let expected = Ok(("", String::from("Add(Value(1), Value(2))"))); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 12 + 6 - 4+ 3"; - let expected = Ok(( - "", - String::from("Add(Sub(Add(Value(12), Value(6)), Value(4)), Value(3))"), - )); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 1 + 2*3 + 4"; - let expected = Ok(( - "", - String::from("Add(Add(Value(1), Mul(Value(2), Value(3))), Value(4))"), - )); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); -} - -#[test] -fn parens_test() { - let input = " ( 2 )"; - let expected = Ok(("", String::from("Paren(Value(2))"))); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 2* ( 3 + 4 ) "; - let expected = Ok(( - "", - String::from("Mul(Value(2), Paren(Add(Value(3), Value(4))))"), - )); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); - - let input = " 2*2 / ( 5 - 1) + 3"; - let expected = Ok(( - "", - String::from("Add(Div(Mul(Value(2), Value(2)), Paren(Sub(Value(5), Value(1)))), Value(3))"), - )); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); -} diff --git a/examples/arithmetic/parser_lexer.rs b/examples/arithmetic/parser_lexer.rs index 8bc2c685..f9963b6f 100644 --- a/examples/arithmetic/parser_lexer.rs +++ b/examples/arithmetic/parser_lexer.rs @@ -145,7 +145,7 @@ pub(crate) fn expr(i: &mut &[Token]) -> Result { .parse_next(i) } -fn term(i: &mut &[Token]) -> Result { +pub(crate) fn term(i: &mut &[Token]) -> Result { let init = factor.parse_next(i)?; repeat( @@ -168,7 +168,7 @@ fn term(i: &mut &[Token]) -> Result { .parse_next(i) } -fn factor(i: &mut &[Token]) -> Result { +pub(crate) fn factor(i: &mut &[Token]) -> Result { alt(( one_of(|t| matches!(t, Token::Value(_))).map(|t| match t { Token::Value(v) => Expr::Value(v), @@ -184,118 +184,3 @@ fn parens(i: &mut &[Token]) -> Result { .map(|e| Expr::Paren(Box::new(e))) .parse_next(i) } - -#[test] -fn lex_test() { - let input = "3"; - let expected = Ok(String::from(r#"("", [Value(3)])"#)); - assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); - - let input = " 24 "; - let expected = Ok(String::from(r#"("", [Value(24)])"#)); - assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); - - let input = " 12 *2 / 3"; - let expected = Ok(String::from( - r#"("", [Value(12), Oper(Mul), Value(2), Oper(Div), Value(3)])"#, - )); - assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); - - let input = " 2*2 / ( 5 - 1) + 3"; - let expected = Ok(String::from( - r#"("", [Value(2), Oper(Mul), Value(2), Oper(Div), OpenParen, Value(5), Oper(Sub), Value(1), CloseParen, Oper(Add), Value(3)])"#, - )); - assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); -} - -#[test] -fn factor_test() { - let input = "3"; - let expected = Ok(String::from("Value(3)")); - let input = lex.parse(input).unwrap(); - assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 12"; - let expected = Ok(String::from("Value(12)")); - let input = lex.parse(input).unwrap(); - assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = "537 "; - let expected = Ok(String::from("Value(537)")); - let input = lex.parse(input).unwrap(); - assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 24 "; - let expected = Ok(String::from("Value(24)")); - let input = lex.parse(input).unwrap(); - assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); -} - -#[test] -fn term_test() { - let input = " 12 *2 / 3"; - let expected = Ok(String::from("Div(Mul(Value(12), Value(2)), Value(3))")); - let input = lex.parse(input).unwrap(); - assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 12 *2 / 3"; - let expected = Ok(String::from("Div(Mul(Value(12), Value(2)), Value(3))")); - let input = lex.parse(input).unwrap(); - assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 2* 3 *2 *2 / 3"; - let expected = Ok(String::from( - "Div(Mul(Mul(Mul(Value(2), Value(3)), Value(2)), Value(2)), Value(3))", - )); - let input = lex.parse(input).unwrap(); - assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 48 / 3/2"; - let expected = Ok(String::from("Div(Div(Value(48), Value(3)), Value(2))")); - let input = lex.parse(input).unwrap(); - assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); -} - -#[test] -fn expr_test() { - let input = " 1 + 2 "; - let expected = Ok(String::from("Add(Value(1), Value(2))")); - let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 12 + 6 - 4+ 3"; - let expected = Ok(String::from( - "Add(Sub(Add(Value(12), Value(6)), Value(4)), Value(3))", - )); - let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 1 + 2*3 + 4"; - let expected = Ok(String::from( - "Add(Add(Value(1), Mul(Value(2), Value(3))), Value(4))", - )); - let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); -} - -#[test] -fn parens_test() { - let input = " ( 2 )"; - let expected = Ok(String::from("Paren(Value(2))")); - let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 2* ( 3 + 4 ) "; - let expected = Ok(String::from( - "Mul(Value(2), Paren(Add(Value(3), Value(4))))", - )); - let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); - - let input = " 2*2 / ( 5 - 1) + 3"; - let expected = Ok(String::from( - "Add(Div(Mul(Value(2), Value(2)), Paren(Sub(Value(5), Value(1)))), Value(3))", - )); - let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); -} diff --git a/examples/arithmetic/test_parser.rs b/examples/arithmetic/test_parser.rs new file mode 100644 index 00000000..d50f4c13 --- /dev/null +++ b/examples/arithmetic/test_parser.rs @@ -0,0 +1,71 @@ +use winnow::prelude::*; + +use crate::parser::*; + +#[test] +fn factor_test() { + let input = "3"; + let expected = Ok(("", 3)); + assert_eq!(factor.parse_peek(input), expected); + + let input = " 12"; + let expected = Ok(("", 12)); + assert_eq!(factor.parse_peek(input), expected); + + let input = "537 "; + let expected = Ok(("", 537)); + assert_eq!(factor.parse_peek(input), expected); + + let input = " 24 "; + let expected = Ok(("", 24)); + assert_eq!(factor.parse_peek(input), expected); +} + +#[test] +fn term_test() { + let input = " 12 *2 / 3"; + let expected = Ok(("", 8)); + assert_eq!(term.parse_peek(input), expected); + + let input = " 12 *2 / 3"; + let expected = Ok(("", 8)); + assert_eq!(term.parse_peek(input), expected); + + let input = " 2* 3 *2 *2 / 3"; + let expected = Ok(("", 8)); + assert_eq!(term.parse_peek(input), expected); + + let input = " 48 / 3/2"; + let expected = Ok(("", 8)); + assert_eq!(term.parse_peek(input), expected); +} + +#[test] +fn expr_test() { + let input = " 1 + 2 "; + let expected = Ok(("", 3)); + assert_eq!(expr.parse_peek(input), expected); + + let input = " 12 + 6 - 4+ 3"; + let expected = Ok(("", 17)); + assert_eq!(expr.parse_peek(input), expected); + + let input = " 1 + 2*3 + 4"; + let expected = Ok(("", 11)); + assert_eq!(expr.parse_peek(input), expected); +} + +#[test] +fn parens_test() { + let input = " ( 2 )"; + let expected = Ok(("", 2)); + assert_eq!(expr.parse_peek(input), expected); + + let input = " 2* ( 3 + 4 ) "; + let expected = Ok(("", 14)); + assert_eq!(expr.parse_peek(input), expected); + + let input = " 2*2 / ( 5 - 1) + 3"; + let expected = Ok(("", 4)); + assert_eq!(expr.parse_peek(input), expected); +} diff --git a/examples/arithmetic/test_parser_ast.rs b/examples/arithmetic/test_parser_ast.rs new file mode 100644 index 00000000..1e00ef7c --- /dev/null +++ b/examples/arithmetic/test_parser_ast.rs @@ -0,0 +1,86 @@ +use winnow::prelude::*; + +use crate::parser_ast::*; + +#[test] +fn factor_test() { + let input = "3"; + let expected = Ok(("", String::from("Value(3)"))); + assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); + + let input = " 12"; + let expected = Ok(("", String::from("Value(12)"))); + assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); + + let input = "537 "; + let expected = Ok(("", String::from("Value(537)"))); + assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); + + let input = " 24 "; + let expected = Ok(("", String::from("Value(24)"))); + assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); +} + +#[test] +fn term_test() { + let input = " 12 *2 / 3"; + let expected = Ok(("", String::from("Div(Mul(Value(12), Value(2)), Value(3))"))); + assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); + + let input = " 12 *2 / 3"; + let expected = Ok(("", String::from("Div(Mul(Value(12), Value(2)), Value(3))"))); + assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); + + let input = " 2* 3 *2 *2 / 3"; + let expected = Ok(( + "", + String::from("Div(Mul(Mul(Mul(Value(2), Value(3)), Value(2)), Value(2)), Value(3))"), + )); + assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); + + let input = " 48 / 3/2"; + let expected = Ok(("", String::from("Div(Div(Value(48), Value(3)), Value(2))"))); + assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); +} + +#[test] +fn expr_test() { + let input = " 1 + 2 "; + let expected = Ok(("", String::from("Add(Value(1), Value(2))"))); + assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); + + let input = " 12 + 6 - 4+ 3"; + let expected = Ok(( + "", + String::from("Add(Sub(Add(Value(12), Value(6)), Value(4)), Value(3))"), + )); + assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); + + let input = " 1 + 2*3 + 4"; + let expected = Ok(( + "", + String::from("Add(Add(Value(1), Mul(Value(2), Value(3))), Value(4))"), + )); + assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); +} + +#[test] +fn parens_test() { + let input = " ( 2 )"; + let expected = Ok(("", String::from("Paren(Value(2))"))); + assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); + + let input = " 2* ( 3 + 4 ) "; + let expected = Ok(( + "", + String::from("Mul(Value(2), Paren(Add(Value(3), Value(4))))"), + )); + assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); + + let input = " 2*2 / ( 5 - 1) + 3"; + let expected = Ok(( + "", + String::from("Add(Div(Mul(Value(2), Value(2)), Paren(Sub(Value(5), Value(1)))), Value(3))"), + )); + assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); +} diff --git a/examples/arithmetic/test_parser_lexer.rs b/examples/arithmetic/test_parser_lexer.rs new file mode 100644 index 00000000..a90c50da --- /dev/null +++ b/examples/arithmetic/test_parser_lexer.rs @@ -0,0 +1,118 @@ +use winnow::prelude::*; + +use crate::parser_lexer::*; + +#[test] +fn lex_test() { + let input = "3"; + let expected = Ok(String::from(r#"("", [Value(3)])"#)); + assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); + + let input = " 24 "; + let expected = Ok(String::from(r#"("", [Value(24)])"#)); + assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); + + let input = " 12 *2 / 3"; + let expected = Ok(String::from( + r#"("", [Value(12), Oper(Mul), Value(2), Oper(Div), Value(3)])"#, + )); + assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); + + let input = " 2*2 / ( 5 - 1) + 3"; + let expected = Ok(String::from( + r#"("", [Value(2), Oper(Mul), Value(2), Oper(Div), OpenParen, Value(5), Oper(Sub), Value(1), CloseParen, Oper(Add), Value(3)])"#, + )); + assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); +} + +#[test] +fn factor_test() { + let input = "3"; + let expected = Ok(String::from("Value(3)")); + let input = lex.parse(input).unwrap(); + assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); + + let input = " 12"; + let expected = Ok(String::from("Value(12)")); + let input = lex.parse(input).unwrap(); + assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); + + let input = "537 "; + let expected = Ok(String::from("Value(537)")); + let input = lex.parse(input).unwrap(); + assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); + + let input = " 24 "; + let expected = Ok(String::from("Value(24)")); + let input = lex.parse(input).unwrap(); + assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); +} + +#[test] +fn term_test() { + let input = " 12 *2 / 3"; + let expected = Ok(String::from("Div(Mul(Value(12), Value(2)), Value(3))")); + let input = lex.parse(input).unwrap(); + assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); + + let input = " 12 *2 / 3"; + let expected = Ok(String::from("Div(Mul(Value(12), Value(2)), Value(3))")); + let input = lex.parse(input).unwrap(); + assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); + + let input = " 2* 3 *2 *2 / 3"; + let expected = Ok(String::from( + "Div(Mul(Mul(Mul(Value(2), Value(3)), Value(2)), Value(2)), Value(3))", + )); + let input = lex.parse(input).unwrap(); + assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); + + let input = " 48 / 3/2"; + let expected = Ok(String::from("Div(Div(Value(48), Value(3)), Value(2))")); + let input = lex.parse(input).unwrap(); + assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); +} + +#[test] +fn expr_test() { + let input = " 1 + 2 "; + let expected = Ok(String::from("Add(Value(1), Value(2))")); + let input = lex.parse(input).unwrap(); + assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); + + let input = " 12 + 6 - 4+ 3"; + let expected = Ok(String::from( + "Add(Sub(Add(Value(12), Value(6)), Value(4)), Value(3))", + )); + let input = lex.parse(input).unwrap(); + assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); + + let input = " 1 + 2*3 + 4"; + let expected = Ok(String::from( + "Add(Add(Value(1), Mul(Value(2), Value(3))), Value(4))", + )); + let input = lex.parse(input).unwrap(); + assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); +} + +#[test] +fn parens_test() { + let input = " ( 2 )"; + let expected = Ok(String::from("Paren(Value(2))")); + let input = lex.parse(input).unwrap(); + assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); + + let input = " 2* ( 3 + 4 ) "; + let expected = Ok(String::from( + "Mul(Value(2), Paren(Add(Value(3), Value(4))))", + )); + let input = lex.parse(input).unwrap(); + assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); + + let input = " 2*2 / ( 5 - 1) + 3"; + let expected = Ok(String::from( + "Add(Div(Mul(Value(2), Value(2)), Paren(Sub(Value(5), Value(1)))), Value(3))", + )); + let input = lex.parse(input).unwrap(); + assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); +} From e9bfd101eb1b8ef3f17644b2cf1455e1e5d4e61c Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 28 Feb 2025 10:57:06 -0600 Subject: [PATCH 02/11] test(examples): Switch to snapbox --- examples/arithmetic/test_parser.rs | 171 ++++++++-- examples/arithmetic/test_parser_ast.rs | 293 ++++++++++++++--- examples/arithmetic/test_parser_lexer.rs | 394 ++++++++++++++++++++--- 3 files changed, 737 insertions(+), 121 deletions(-) diff --git a/examples/arithmetic/test_parser.rs b/examples/arithmetic/test_parser.rs index d50f4c13..f22ba62b 100644 --- a/examples/arithmetic/test_parser.rs +++ b/examples/arithmetic/test_parser.rs @@ -1,3 +1,6 @@ +use snapbox::assert_data_eq; +use snapbox::prelude::*; +use snapbox::str; use winnow::prelude::*; use crate::parser::*; @@ -5,67 +8,179 @@ use crate::parser::*; #[test] fn factor_test() { let input = "3"; - let expected = Ok(("", 3)); - assert_eq!(factor.parse_peek(input), expected); + let expected = str![[r#" +Ok( + ( + "", + 3, + ), +) + +"#]]; + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); let input = " 12"; - let expected = Ok(("", 12)); - assert_eq!(factor.parse_peek(input), expected); + let expected = str![[r#" +Ok( + ( + "", + 12, + ), +) + +"#]]; + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); let input = "537 "; - let expected = Ok(("", 537)); - assert_eq!(factor.parse_peek(input), expected); + let expected = str![[r#" +Ok( + ( + "", + 537, + ), +) + +"#]]; + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); let input = " 24 "; - let expected = Ok(("", 24)); - assert_eq!(factor.parse_peek(input), expected); + let expected = str![[r#" +Ok( + ( + "", + 24, + ), +) + +"#]]; + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); } #[test] fn term_test() { let input = " 12 *2 / 3"; - let expected = Ok(("", 8)); - assert_eq!(term.parse_peek(input), expected); + let expected = str![[r#" +Ok( + ( + "", + 8, + ), +) + +"#]]; + assert_data_eq!(term.parse_peek(input).to_debug(), expected); let input = " 12 *2 / 3"; - let expected = Ok(("", 8)); - assert_eq!(term.parse_peek(input), expected); + let expected = str![[r#" +Ok( + ( + "", + 8, + ), +) + +"#]]; + assert_data_eq!(term.parse_peek(input).to_debug(), expected); let input = " 2* 3 *2 *2 / 3"; - let expected = Ok(("", 8)); - assert_eq!(term.parse_peek(input), expected); + let expected = str![[r#" +Ok( + ( + "", + 8, + ), +) + +"#]]; + assert_data_eq!(term.parse_peek(input).to_debug(), expected); let input = " 48 / 3/2"; - let expected = Ok(("", 8)); - assert_eq!(term.parse_peek(input), expected); + let expected = str![[r#" +Ok( + ( + "", + 8, + ), +) + +"#]]; + assert_data_eq!(term.parse_peek(input).to_debug(), expected); } #[test] fn expr_test() { let input = " 1 + 2 "; - let expected = Ok(("", 3)); - assert_eq!(expr.parse_peek(input), expected); + let expected = str![[r#" +Ok( + ( + "", + 3, + ), +) + +"#]]; + assert_data_eq!(expr.parse_peek(input).to_debug(), expected); let input = " 12 + 6 - 4+ 3"; - let expected = Ok(("", 17)); - assert_eq!(expr.parse_peek(input), expected); + let expected = str![[r#" +Ok( + ( + "", + 17, + ), +) + +"#]]; + assert_data_eq!(expr.parse_peek(input).to_debug(), expected); let input = " 1 + 2*3 + 4"; - let expected = Ok(("", 11)); - assert_eq!(expr.parse_peek(input), expected); + let expected = str![[r#" +Ok( + ( + "", + 11, + ), +) + +"#]]; + assert_data_eq!(expr.parse_peek(input).to_debug(), expected); } #[test] fn parens_test() { let input = " ( 2 )"; - let expected = Ok(("", 2)); - assert_eq!(expr.parse_peek(input), expected); + let expected = str![[r#" +Ok( + ( + "", + 2, + ), +) + +"#]]; + assert_data_eq!(expr.parse_peek(input).to_debug(), expected); let input = " 2* ( 3 + 4 ) "; - let expected = Ok(("", 14)); - assert_eq!(expr.parse_peek(input), expected); + let expected = str![[r#" +Ok( + ( + "", + 14, + ), +) + +"#]]; + assert_data_eq!(expr.parse_peek(input).to_debug(), expected); let input = " 2*2 / ( 5 - 1) + 3"; - let expected = Ok(("", 4)); - assert_eq!(expr.parse_peek(input), expected); + let expected = str![[r#" +Ok( + ( + "", + 4, + ), +) + +"#]]; + assert_data_eq!(expr.parse_peek(input).to_debug(), expected); } diff --git a/examples/arithmetic/test_parser_ast.rs b/examples/arithmetic/test_parser_ast.rs index 1e00ef7c..cb99af87 100644 --- a/examples/arithmetic/test_parser_ast.rs +++ b/examples/arithmetic/test_parser_ast.rs @@ -1,3 +1,6 @@ +use snapbox::assert_data_eq; +use snapbox::prelude::*; +use snapbox::str; use winnow::prelude::*; use crate::parser_ast::*; @@ -5,82 +8,286 @@ use crate::parser_ast::*; #[test] fn factor_test() { let input = "3"; - let expected = Ok(("", String::from("Value(3)"))); - assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); + let expected = str![[r#" +Ok( + Value( + 3, + ), +) + +"#]]; + assert_data_eq!(factor.parse(input).to_debug(), expected); let input = " 12"; - let expected = Ok(("", String::from("Value(12)"))); - assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); + let expected = str![[r#" +Ok( + Value( + 12, + ), +) + +"#]]; + assert_data_eq!(factor.parse(input).to_debug(), expected); let input = "537 "; - let expected = Ok(("", String::from("Value(537)"))); - assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); + let expected = str![[r#" +Ok( + Value( + 537, + ), +) + +"#]]; + assert_data_eq!(factor.parse(input).to_debug(), expected); let input = " 24 "; - let expected = Ok(("", String::from("Value(24)"))); - assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected); + let expected = str![[r#" +Ok( + Value( + 24, + ), +) + +"#]]; + assert_data_eq!(factor.parse(input).to_debug(), expected); } #[test] fn term_test() { let input = " 12 *2 / 3"; - let expected = Ok(("", String::from("Div(Mul(Value(12), Value(2)), Value(3))"))); - assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); + let expected = str![[r#" +Ok( + Div( + Mul( + Value( + 12, + ), + Value( + 2, + ), + ), + Value( + 3, + ), + ), +) + +"#]]; + assert_data_eq!(term.parse(input).to_debug(), expected); let input = " 12 *2 / 3"; - let expected = Ok(("", String::from("Div(Mul(Value(12), Value(2)), Value(3))"))); - assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); + let expected = str![[r#" +Ok( + Div( + Mul( + Value( + 12, + ), + Value( + 2, + ), + ), + Value( + 3, + ), + ), +) + +"#]]; + assert_data_eq!(term.parse(input).to_debug(), expected); let input = " 2* 3 *2 *2 / 3"; - let expected = Ok(( - "", - String::from("Div(Mul(Mul(Mul(Value(2), Value(3)), Value(2)), Value(2)), Value(3))"), - )); - assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); + let expected = str![[r#" +Ok( + Div( + Mul( + Mul( + Mul( + Value( + 2, + ), + Value( + 3, + ), + ), + Value( + 2, + ), + ), + Value( + 2, + ), + ), + Value( + 3, + ), + ), +) + +"#]]; + assert_data_eq!(term.parse(input).to_debug(), expected); let input = " 48 / 3/2"; - let expected = Ok(("", String::from("Div(Div(Value(48), Value(3)), Value(2))"))); - assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected); + let expected = str![[r#" +Ok( + Div( + Div( + Value( + 48, + ), + Value( + 3, + ), + ), + Value( + 2, + ), + ), +) + +"#]]; + assert_data_eq!(term.parse(input).to_debug(), expected); } #[test] fn expr_test() { let input = " 1 + 2 "; - let expected = Ok(("", String::from("Add(Value(1), Value(2))"))); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); + let expected = str![[r#" +Ok( + Add( + Value( + 1, + ), + Value( + 2, + ), + ), +) + +"#]]; + assert_data_eq!(expr.parse(input).to_debug(), expected); let input = " 12 + 6 - 4+ 3"; - let expected = Ok(( - "", - String::from("Add(Sub(Add(Value(12), Value(6)), Value(4)), Value(3))"), - )); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); + let expected = str![[r#" +Ok( + Add( + Sub( + Add( + Value( + 12, + ), + Value( + 6, + ), + ), + Value( + 4, + ), + ), + Value( + 3, + ), + ), +) + +"#]]; + assert_data_eq!(expr.parse(input).to_debug(), expected); let input = " 1 + 2*3 + 4"; - let expected = Ok(( - "", - String::from("Add(Add(Value(1), Mul(Value(2), Value(3))), Value(4))"), - )); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); + let expected = str![[r#" +Ok( + Add( + Add( + Value( + 1, + ), + Mul( + Value( + 2, + ), + Value( + 3, + ), + ), + ), + Value( + 4, + ), + ), +) + +"#]]; + assert_data_eq!(expr.parse(input).to_debug(), expected); } #[test] fn parens_test() { let input = " ( 2 )"; - let expected = Ok(("", String::from("Paren(Value(2))"))); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); + let expected = str![[r#" +Ok( + Paren( + Value( + 2, + ), + ), +) + +"#]]; + assert_data_eq!(expr.parse(input).to_debug(), expected); let input = " 2* ( 3 + 4 ) "; - let expected = Ok(( - "", - String::from("Mul(Value(2), Paren(Add(Value(3), Value(4))))"), - )); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); + let expected = str![[r#" +Ok( + Mul( + Value( + 2, + ), + Paren( + Add( + Value( + 3, + ), + Value( + 4, + ), + ), + ), + ), +) + +"#]]; + assert_data_eq!(expr.parse(input).to_debug(), expected); let input = " 2*2 / ( 5 - 1) + 3"; - let expected = Ok(( - "", - String::from("Add(Div(Mul(Value(2), Value(2)), Paren(Sub(Value(5), Value(1)))), Value(3))"), - )); - assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected); + let expected = str![[r#" +Ok( + Add( + Div( + Mul( + Value( + 2, + ), + Value( + 2, + ), + ), + Paren( + Sub( + Value( + 5, + ), + Value( + 1, + ), + ), + ), + ), + Value( + 3, + ), + ), +) + +"#]]; + assert_data_eq!(expr.parse(input).to_debug(), expected); } diff --git a/examples/arithmetic/test_parser_lexer.rs b/examples/arithmetic/test_parser_lexer.rs index a90c50da..de19ecf2 100644 --- a/examples/arithmetic/test_parser_lexer.rs +++ b/examples/arithmetic/test_parser_lexer.rs @@ -1,3 +1,6 @@ +use snapbox::assert_data_eq; +use snapbox::prelude::*; +use snapbox::str; use winnow::prelude::*; use crate::parser_lexer::*; @@ -5,114 +8,405 @@ use crate::parser_lexer::*; #[test] fn lex_test() { let input = "3"; - let expected = Ok(String::from(r#"("", [Value(3)])"#)); - assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); + let expected = str![[r#" +Ok( + ( + "", + [ + Value( + 3, + ), + ], + ), +) + +"#]]; + assert_data_eq!(lex.parse_peek(input).to_debug(), expected); let input = " 24 "; - let expected = Ok(String::from(r#"("", [Value(24)])"#)); - assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); + let expected = str![[r#" +Ok( + ( + "", + [ + Value( + 24, + ), + ], + ), +) + +"#]]; + assert_data_eq!(lex.parse_peek(input).to_debug(), expected); let input = " 12 *2 / 3"; - let expected = Ok(String::from( - r#"("", [Value(12), Oper(Mul), Value(2), Oper(Div), Value(3)])"#, - )); - assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); + let expected = str![[r#" +Ok( + ( + "", + [ + Value( + 12, + ), + Oper( + Mul, + ), + Value( + 2, + ), + Oper( + Div, + ), + Value( + 3, + ), + ], + ), +) + +"#]]; + assert_data_eq!(lex.parse_peek(input).to_debug(), expected); let input = " 2*2 / ( 5 - 1) + 3"; - let expected = Ok(String::from( - r#"("", [Value(2), Oper(Mul), Value(2), Oper(Div), OpenParen, Value(5), Oper(Sub), Value(1), CloseParen, Oper(Add), Value(3)])"#, - )); - assert_eq!(lex.parse_peek(input).map(|e| format!("{e:?}")), expected); + let expected = str![[r#" +Ok( + ( + "", + [ + Value( + 2, + ), + Oper( + Mul, + ), + Value( + 2, + ), + Oper( + Div, + ), + OpenParen, + Value( + 5, + ), + Oper( + Sub, + ), + Value( + 1, + ), + CloseParen, + Oper( + Add, + ), + Value( + 3, + ), + ], + ), +) + +"#]]; + assert_data_eq!(lex.parse_peek(input).to_debug(), expected); } #[test] fn factor_test() { let input = "3"; - let expected = Ok(String::from("Value(3)")); + let expected = str![[r#" +Ok( + Value( + 3, + ), +) + +"#]]; let input = lex.parse(input).unwrap(); - assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); + assert_data_eq!(factor.parse(&input).to_debug(), expected); let input = " 12"; - let expected = Ok(String::from("Value(12)")); + let expected = str![[r#" +Ok( + Value( + 12, + ), +) + +"#]]; let input = lex.parse(input).unwrap(); - assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); + assert_data_eq!(factor.parse(&input).to_debug(), expected); let input = "537 "; - let expected = Ok(String::from("Value(537)")); + let expected = str![[r#" +Ok( + Value( + 537, + ), +) + +"#]]; let input = lex.parse(input).unwrap(); - assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); + assert_data_eq!(factor.parse(&input).to_debug(), expected); let input = " 24 "; - let expected = Ok(String::from("Value(24)")); + let expected = str![[r#" +Ok( + Value( + 24, + ), +) + +"#]]; let input = lex.parse(input).unwrap(); - assert_eq!(factor.map(|e| format!("{e:?}")).parse(&input), expected); + assert_data_eq!(factor.parse(&input).to_debug(), expected); } #[test] fn term_test() { let input = " 12 *2 / 3"; - let expected = Ok(String::from("Div(Mul(Value(12), Value(2)), Value(3))")); + let expected = str![[r#" +Ok( + Div( + Mul( + Value( + 12, + ), + Value( + 2, + ), + ), + Value( + 3, + ), + ), +) + +"#]]; let input = lex.parse(input).unwrap(); - assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); + assert_data_eq!(term.parse(&input).to_debug(), expected); let input = " 12 *2 / 3"; - let expected = Ok(String::from("Div(Mul(Value(12), Value(2)), Value(3))")); + let expected = str![[r#" +Ok( + Div( + Mul( + Value( + 12, + ), + Value( + 2, + ), + ), + Value( + 3, + ), + ), +) + +"#]]; let input = lex.parse(input).unwrap(); - assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); + assert_data_eq!(term.parse(&input).to_debug(), expected); let input = " 2* 3 *2 *2 / 3"; - let expected = Ok(String::from( - "Div(Mul(Mul(Mul(Value(2), Value(3)), Value(2)), Value(2)), Value(3))", - )); + let expected = str![[r#" +Ok( + Div( + Mul( + Mul( + Mul( + Value( + 2, + ), + Value( + 3, + ), + ), + Value( + 2, + ), + ), + Value( + 2, + ), + ), + Value( + 3, + ), + ), +) + +"#]]; let input = lex.parse(input).unwrap(); - assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); + assert_data_eq!(term.parse(&input).to_debug(), expected); let input = " 48 / 3/2"; - let expected = Ok(String::from("Div(Div(Value(48), Value(3)), Value(2))")); + let expected = str![[r#" +Ok( + Div( + Div( + Value( + 48, + ), + Value( + 3, + ), + ), + Value( + 2, + ), + ), +) + +"#]]; let input = lex.parse(input).unwrap(); - assert_eq!(term.map(|e| format!("{e:?}")).parse(&input), expected); + assert_data_eq!(term.parse(&input).to_debug(), expected); } #[test] fn expr_test() { let input = " 1 + 2 "; - let expected = Ok(String::from("Add(Value(1), Value(2))")); + let expected = str![[r#" +Ok( + Add( + Value( + 1, + ), + Value( + 2, + ), + ), +) + +"#]]; let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); + assert_data_eq!(expr.parse(&input).to_debug(), expected); let input = " 12 + 6 - 4+ 3"; - let expected = Ok(String::from( - "Add(Sub(Add(Value(12), Value(6)), Value(4)), Value(3))", - )); + let expected = str![[r#" +Ok( + Add( + Sub( + Add( + Value( + 12, + ), + Value( + 6, + ), + ), + Value( + 4, + ), + ), + Value( + 3, + ), + ), +) + +"#]]; let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); + assert_data_eq!(expr.parse(&input).to_debug(), expected); let input = " 1 + 2*3 + 4"; - let expected = Ok(String::from( - "Add(Add(Value(1), Mul(Value(2), Value(3))), Value(4))", - )); + let expected = str![[r#" +Ok( + Add( + Add( + Value( + 1, + ), + Mul( + Value( + 2, + ), + Value( + 3, + ), + ), + ), + Value( + 4, + ), + ), +) + +"#]]; let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); + assert_data_eq!(expr.parse(&input).to_debug(), expected); } #[test] fn parens_test() { let input = " ( 2 )"; - let expected = Ok(String::from("Paren(Value(2))")); + let expected = str![[r#" +Ok( + Paren( + Value( + 2, + ), + ), +) + +"#]]; let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); + assert_data_eq!(expr.parse(&input).to_debug(), expected); let input = " 2* ( 3 + 4 ) "; - let expected = Ok(String::from( - "Mul(Value(2), Paren(Add(Value(3), Value(4))))", - )); + let expected = str![[r#" +Ok( + Mul( + Value( + 2, + ), + Paren( + Add( + Value( + 3, + ), + Value( + 4, + ), + ), + ), + ), +) + +"#]]; let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); + assert_data_eq!(expr.parse(&input).to_debug(), expected); let input = " 2*2 / ( 5 - 1) + 3"; - let expected = Ok(String::from( - "Add(Div(Mul(Value(2), Value(2)), Paren(Sub(Value(5), Value(1)))), Value(3))", - )); + let expected = str![[r#" +Ok( + Add( + Div( + Mul( + Value( + 2, + ), + Value( + 2, + ), + ), + Paren( + Sub( + Value( + 5, + ), + Value( + 1, + ), + ), + ), + ), + Value( + 3, + ), + ), +) + +"#]]; let input = lex.parse(input).unwrap(); - assert_eq!(expr.map(|e| format!("{e:?}")).parse(&input), expected); + assert_data_eq!(expr.parse(&input).to_debug(), expected); } From 00787d62e459cc90b97132b32f8bab4d98885234 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 28 Feb 2025 11:15:31 -0600 Subject: [PATCH 03/11] test(examples): Don't assume full content on parsers --- examples/arithmetic/test_parser_ast.rs | 120 ++++++++++++++--------- examples/arithmetic/test_parser_lexer.rs | 120 ++++++++++++++--------- 2 files changed, 144 insertions(+), 96 deletions(-) diff --git a/examples/arithmetic/test_parser_ast.rs b/examples/arithmetic/test_parser_ast.rs index cb99af87..86c08be7 100644 --- a/examples/arithmetic/test_parser_ast.rs +++ b/examples/arithmetic/test_parser_ast.rs @@ -10,46 +10,58 @@ fn factor_test() { let input = "3"; let expected = str![[r#" Ok( - Value( - 3, + ( + "", + Value( + 3, + ), ), ) "#]]; - assert_data_eq!(factor.parse(input).to_debug(), expected); + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); let input = " 12"; let expected = str![[r#" Ok( - Value( - 12, + ( + "", + Value( + 12, + ), ), ) "#]]; - assert_data_eq!(factor.parse(input).to_debug(), expected); + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); let input = "537 "; let expected = str![[r#" Ok( - Value( - 537, + ( + "", + Value( + 537, + ), ), ) "#]]; - assert_data_eq!(factor.parse(input).to_debug(), expected); + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); let input = " 24 "; let expected = str![[r#" Ok( - Value( - 24, + ( + "", + Value( + 24, + ), ), ) "#]]; - assert_data_eq!(factor.parse(input).to_debug(), expected); + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); } #[test] @@ -57,57 +69,69 @@ fn term_test() { let input = " 12 *2 / 3"; let expected = str![[r#" Ok( - Div( - Mul( - Value( - 12, + ( + "", + Div( + Mul( + Value( + 12, + ), + Value( + 2, + ), ), Value( - 2, + 3, ), ), - Value( - 3, - ), ), ) "#]]; - assert_data_eq!(term.parse(input).to_debug(), expected); + assert_data_eq!(term.parse_peek(input).to_debug(), expected); let input = " 12 *2 / 3"; let expected = str![[r#" Ok( - Div( - Mul( - Value( - 12, + ( + "", + Div( + Mul( + Value( + 12, + ), + Value( + 2, + ), ), Value( - 2, + 3, ), ), - Value( - 3, - ), ), ) "#]]; - assert_data_eq!(term.parse(input).to_debug(), expected); + assert_data_eq!(term.parse_peek(input).to_debug(), expected); let input = " 2* 3 *2 *2 / 3"; let expected = str![[r#" Ok( - Div( - Mul( + ( + "", + Div( Mul( Mul( - Value( - 2, + Mul( + Value( + 2, + ), + Value( + 3, + ), ), Value( - 3, + 2, ), ), Value( @@ -115,38 +139,38 @@ Ok( ), ), Value( - 2, + 3, ), ), - Value( - 3, - ), ), ) "#]]; - assert_data_eq!(term.parse(input).to_debug(), expected); + assert_data_eq!(term.parse_peek(input).to_debug(), expected); let input = " 48 / 3/2"; let expected = str![[r#" Ok( - Div( + ( + "", Div( - Value( - 48, + Div( + Value( + 48, + ), + Value( + 3, + ), ), Value( - 3, + 2, ), ), - Value( - 2, - ), ), ) "#]]; - assert_data_eq!(term.parse(input).to_debug(), expected); + assert_data_eq!(term.parse_peek(input).to_debug(), expected); } #[test] diff --git a/examples/arithmetic/test_parser_lexer.rs b/examples/arithmetic/test_parser_lexer.rs index de19ecf2..235b6e42 100644 --- a/examples/arithmetic/test_parser_lexer.rs +++ b/examples/arithmetic/test_parser_lexer.rs @@ -115,50 +115,62 @@ fn factor_test() { let input = "3"; let expected = str![[r#" Ok( - Value( - 3, + ( + [], + Value( + 3, + ), ), ) "#]]; let input = lex.parse(input).unwrap(); - assert_data_eq!(factor.parse(&input).to_debug(), expected); + assert_data_eq!(factor.parse_peek(&input).to_debug(), expected); let input = " 12"; let expected = str![[r#" Ok( - Value( - 12, + ( + [], + Value( + 12, + ), ), ) "#]]; let input = lex.parse(input).unwrap(); - assert_data_eq!(factor.parse(&input).to_debug(), expected); + assert_data_eq!(factor.parse_peek(&input).to_debug(), expected); let input = "537 "; let expected = str![[r#" Ok( - Value( - 537, + ( + [], + Value( + 537, + ), ), ) "#]]; let input = lex.parse(input).unwrap(); - assert_data_eq!(factor.parse(&input).to_debug(), expected); + assert_data_eq!(factor.parse_peek(&input).to_debug(), expected); let input = " 24 "; let expected = str![[r#" Ok( - Value( - 24, + ( + [], + Value( + 24, + ), ), ) "#]]; let input = lex.parse(input).unwrap(); - assert_data_eq!(factor.parse(&input).to_debug(), expected); + assert_data_eq!(factor.parse_peek(&input).to_debug(), expected); } #[test] @@ -166,59 +178,71 @@ fn term_test() { let input = " 12 *2 / 3"; let expected = str![[r#" Ok( - Div( - Mul( - Value( - 12, + ( + [], + Div( + Mul( + Value( + 12, + ), + Value( + 2, + ), ), Value( - 2, + 3, ), ), - Value( - 3, - ), ), ) "#]]; let input = lex.parse(input).unwrap(); - assert_data_eq!(term.parse(&input).to_debug(), expected); + assert_data_eq!(term.parse_peek(&input).to_debug(), expected); let input = " 12 *2 / 3"; let expected = str![[r#" Ok( - Div( - Mul( - Value( - 12, + ( + [], + Div( + Mul( + Value( + 12, + ), + Value( + 2, + ), ), Value( - 2, + 3, ), ), - Value( - 3, - ), ), ) "#]]; let input = lex.parse(input).unwrap(); - assert_data_eq!(term.parse(&input).to_debug(), expected); + assert_data_eq!(term.parse_peek(&input).to_debug(), expected); let input = " 2* 3 *2 *2 / 3"; let expected = str![[r#" Ok( - Div( - Mul( + ( + [], + Div( Mul( Mul( - Value( - 2, + Mul( + Value( + 2, + ), + Value( + 3, + ), ), Value( - 3, + 2, ), ), Value( @@ -226,40 +250,40 @@ Ok( ), ), Value( - 2, + 3, ), ), - Value( - 3, - ), ), ) "#]]; let input = lex.parse(input).unwrap(); - assert_data_eq!(term.parse(&input).to_debug(), expected); + assert_data_eq!(term.parse_peek(&input).to_debug(), expected); let input = " 48 / 3/2"; let expected = str![[r#" Ok( - Div( + ( + [], Div( - Value( - 48, + Div( + Value( + 48, + ), + Value( + 3, + ), ), Value( - 3, + 2, ), ), - Value( - 2, - ), ), ) "#]]; let input = lex.parse(input).unwrap(); - assert_data_eq!(term.parse(&input).to_debug(), expected); + assert_data_eq!(term.parse_peek(&input).to_debug(), expected); } #[test] From 3d8aa2703baa7a1794ec3847bd5525ad5aca4040 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 21 Feb 2025 21:25:00 -0600 Subject: [PATCH 04/11] docs(examples): Name the lexer for what it parses --- examples/arithmetic/main.rs | 2 +- examples/arithmetic/parser_lexer.rs | 5 ++-- examples/arithmetic/test_parser_lexer.rs | 36 ++++++++++++------------ 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/examples/arithmetic/main.rs b/examples/arithmetic/main.rs index d262bf19..82873f11 100644 --- a/examples/arithmetic/main.rs +++ b/examples/arithmetic/main.rs @@ -37,7 +37,7 @@ fn calc( println!(" {:#?}={}", result, result.eval()); } Impl::Lexer => { - let tokens = parser_lexer::lex.parse(input)?; + let tokens = parser_lexer::tokens.parse(input)?; println!(" {tokens:#?}"); let result = parser_lexer::expr.parse(tokens.as_slice()).unwrap(); println!(" {:#?}={}", result, result.eval()); diff --git a/examples/arithmetic/parser_lexer.rs b/examples/arithmetic/parser_lexer.rs index f9963b6f..bd5e5e92 100644 --- a/examples/arithmetic/parser_lexer.rs +++ b/examples/arithmetic/parser_lexer.rs @@ -100,11 +100,12 @@ impl winnow::stream::ContainsToken for [Token; LEN] { #[allow(dead_code)] pub(crate) fn expr2(i: &mut &str) -> Result { - let tokens = lex.parse_next(i)?; + let tokens = tokens.parse_next(i)?; expr.parse_next(&mut tokens.as_slice()) } -pub(crate) fn lex(i: &mut &str) -> Result> { +/// Lex tokens +pub(crate) fn tokens(i: &mut &str) -> Result> { preceded(multispaces, repeat(1.., terminated(token, multispaces))).parse_next(i) } diff --git a/examples/arithmetic/test_parser_lexer.rs b/examples/arithmetic/test_parser_lexer.rs index 235b6e42..cebecae8 100644 --- a/examples/arithmetic/test_parser_lexer.rs +++ b/examples/arithmetic/test_parser_lexer.rs @@ -21,7 +21,7 @@ Ok( ) "#]]; - assert_data_eq!(lex.parse_peek(input).to_debug(), expected); + assert_data_eq!(tokens.parse_peek(input).to_debug(), expected); let input = " 24 "; let expected = str![[r#" @@ -37,7 +37,7 @@ Ok( ) "#]]; - assert_data_eq!(lex.parse_peek(input).to_debug(), expected); + assert_data_eq!(tokens.parse_peek(input).to_debug(), expected); let input = " 12 *2 / 3"; let expected = str![[r#" @@ -65,7 +65,7 @@ Ok( ) "#]]; - assert_data_eq!(lex.parse_peek(input).to_debug(), expected); + assert_data_eq!(tokens.parse_peek(input).to_debug(), expected); let input = " 2*2 / ( 5 - 1) + 3"; let expected = str![[r#" @@ -107,7 +107,7 @@ Ok( ) "#]]; - assert_data_eq!(lex.parse_peek(input).to_debug(), expected); + assert_data_eq!(tokens.parse_peek(input).to_debug(), expected); } #[test] @@ -124,7 +124,7 @@ Ok( ) "#]]; - let input = lex.parse(input).unwrap(); + let input = tokens.parse(input).unwrap(); assert_data_eq!(factor.parse_peek(&input).to_debug(), expected); let input = " 12"; @@ -139,7 +139,7 @@ Ok( ) "#]]; - let input = lex.parse(input).unwrap(); + let input = tokens.parse(input).unwrap(); assert_data_eq!(factor.parse_peek(&input).to_debug(), expected); let input = "537 "; @@ -154,7 +154,7 @@ Ok( ) "#]]; - let input = lex.parse(input).unwrap(); + let input = tokens.parse(input).unwrap(); assert_data_eq!(factor.parse_peek(&input).to_debug(), expected); let input = " 24 "; @@ -169,7 +169,7 @@ Ok( ) "#]]; - let input = lex.parse(input).unwrap(); + let input = tokens.parse(input).unwrap(); assert_data_eq!(factor.parse_peek(&input).to_debug(), expected); } @@ -197,7 +197,7 @@ Ok( ) "#]]; - let input = lex.parse(input).unwrap(); + let input = tokens.parse(input).unwrap(); assert_data_eq!(term.parse_peek(&input).to_debug(), expected); let input = " 12 *2 / 3"; @@ -222,7 +222,7 @@ Ok( ) "#]]; - let input = lex.parse(input).unwrap(); + let input = tokens.parse(input).unwrap(); assert_data_eq!(term.parse_peek(&input).to_debug(), expected); let input = " 2* 3 *2 *2 / 3"; @@ -257,7 +257,7 @@ Ok( ) "#]]; - let input = lex.parse(input).unwrap(); + let input = tokens.parse(input).unwrap(); assert_data_eq!(term.parse_peek(&input).to_debug(), expected); let input = " 48 / 3/2"; @@ -282,7 +282,7 @@ Ok( ) "#]]; - let input = lex.parse(input).unwrap(); + let input = tokens.parse(input).unwrap(); assert_data_eq!(term.parse_peek(&input).to_debug(), expected); } @@ -302,7 +302,7 @@ Ok( ) "#]]; - let input = lex.parse(input).unwrap(); + let input = tokens.parse(input).unwrap(); assert_data_eq!(expr.parse(&input).to_debug(), expected); let input = " 12 + 6 - 4+ 3"; @@ -329,7 +329,7 @@ Ok( ) "#]]; - let input = lex.parse(input).unwrap(); + let input = tokens.parse(input).unwrap(); assert_data_eq!(expr.parse(&input).to_debug(), expected); let input = " 1 + 2*3 + 4"; @@ -356,7 +356,7 @@ Ok( ) "#]]; - let input = lex.parse(input).unwrap(); + let input = tokens.parse(input).unwrap(); assert_data_eq!(expr.parse(&input).to_debug(), expected); } @@ -373,7 +373,7 @@ Ok( ) "#]]; - let input = lex.parse(input).unwrap(); + let input = tokens.parse(input).unwrap(); assert_data_eq!(expr.parse(&input).to_debug(), expected); let input = " 2* ( 3 + 4 ) "; @@ -397,7 +397,7 @@ Ok( ) "#]]; - let input = lex.parse(input).unwrap(); + let input = tokens.parse(input).unwrap(); assert_data_eq!(expr.parse(&input).to_debug(), expected); let input = " 2*2 / ( 5 - 1) + 3"; @@ -431,6 +431,6 @@ Ok( ) "#]]; - let input = lex.parse(input).unwrap(); + let input = tokens.parse(input).unwrap(); assert_data_eq!(expr.parse(&input).to_debug(), expected); } From 77fc04fb28c894ff4b1166d6b5c4334e3e398604 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 21 Feb 2025 21:27:15 -0600 Subject: [PATCH 05/11] docs(examples): Group lexer and parser logic --- examples/arithmetic/parser_lexer.rs | 86 +++++++++++++++-------------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/examples/arithmetic/parser_lexer.rs b/examples/arithmetic/parser_lexer.rs index bd5e5e92..b78f6710 100644 --- a/examples/arithmetic/parser_lexer.rs +++ b/examples/arithmetic/parser_lexer.rs @@ -17,41 +17,11 @@ use winnow::{ token::one_of, }; -#[derive(Debug, Clone)] -pub(crate) enum Expr { - Value(i64), - Add(Box, Box), - Sub(Box, Box), - Mul(Box, Box), - Div(Box, Box), - Paren(Box), -} - -impl Expr { - pub(crate) fn eval(&self) -> i64 { - match self { - Self::Value(v) => *v, - Self::Add(lhs, rhs) => lhs.eval() + rhs.eval(), - Self::Sub(lhs, rhs) => lhs.eval() - rhs.eval(), - Self::Mul(lhs, rhs) => lhs.eval() * rhs.eval(), - Self::Div(lhs, rhs) => lhs.eval() / rhs.eval(), - Self::Paren(expr) => expr.eval(), - } - } -} - -impl Display for Expr { - fn fmt(&self, format: &mut Formatter<'_>) -> fmt::Result { - use Expr::{Add, Div, Mul, Paren, Sub, Value}; - match *self { - Value(val) => write!(format, "{val}"), - Add(ref left, ref right) => write!(format, "{left} + {right}"), - Sub(ref left, ref right) => write!(format, "{left} - {right}"), - Mul(ref left, ref right) => write!(format, "{left} * {right}"), - Div(ref left, ref right) => write!(format, "{left} / {right}"), - Paren(ref expr) => write!(format, "({expr})"), - } - } +/// Lex and parse +#[allow(dead_code)] +pub(crate) fn expr2(i: &mut &str) -> Result { + let tokens = tokens.parse_next(i)?; + expr.parse_next(&mut tokens.as_slice()) } #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -98,13 +68,9 @@ impl winnow::stream::ContainsToken for [Token; LEN] { } } -#[allow(dead_code)] -pub(crate) fn expr2(i: &mut &str) -> Result { - let tokens = tokens.parse_next(i)?; - expr.parse_next(&mut tokens.as_slice()) -} - /// Lex tokens +/// +/// See [`expr`] to parse the tokens pub(crate) fn tokens(i: &mut &str) -> Result> { preceded(multispaces, repeat(1.., terminated(token, multispaces))).parse_next(i) } @@ -123,6 +89,44 @@ fn token(i: &mut &str) -> Result { .parse_next(i) } +#[derive(Debug, Clone)] +pub(crate) enum Expr { + Value(i64), + Add(Box, Box), + Sub(Box, Box), + Mul(Box, Box), + Div(Box, Box), + Paren(Box), +} + +impl Expr { + pub(crate) fn eval(&self) -> i64 { + match self { + Self::Value(v) => *v, + Self::Add(lhs, rhs) => lhs.eval() + rhs.eval(), + Self::Sub(lhs, rhs) => lhs.eval() - rhs.eval(), + Self::Mul(lhs, rhs) => lhs.eval() * rhs.eval(), + Self::Div(lhs, rhs) => lhs.eval() / rhs.eval(), + Self::Paren(expr) => expr.eval(), + } + } +} + +impl Display for Expr { + fn fmt(&self, format: &mut Formatter<'_>) -> fmt::Result { + use Expr::{Add, Div, Mul, Paren, Sub, Value}; + match *self { + Value(val) => write!(format, "{val}"), + Add(ref left, ref right) => write!(format, "{left} + {right}"), + Sub(ref left, ref right) => write!(format, "{left} - {right}"), + Mul(ref left, ref right) => write!(format, "{left} * {right}"), + Div(ref left, ref right) => write!(format, "{left} / {right}"), + Paren(ref expr) => write!(format, "({expr})"), + } + } +} + +/// Parse the tokens lexed in [`tokens`] pub(crate) fn expr(i: &mut &[Token]) -> Result { let init = term.parse_next(i)?; From 607c094ad1a08897d941333de1f44676691d2286 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 21 Feb 2025 21:38:20 -0600 Subject: [PATCH 06/11] docs(examples): Use TokenSlice --- examples/arithmetic/main.rs | 3 +- examples/arithmetic/parser_lexer.rs | 60 ++++---- examples/arithmetic/test_parser_lexer.rs | 174 ++++++++++++++++++++--- 3 files changed, 190 insertions(+), 47 deletions(-) diff --git a/examples/arithmetic/main.rs b/examples/arithmetic/main.rs index 82873f11..c1a3a7d0 100644 --- a/examples/arithmetic/main.rs +++ b/examples/arithmetic/main.rs @@ -39,7 +39,8 @@ fn calc( Impl::Lexer => { let tokens = parser_lexer::tokens.parse(input)?; println!(" {tokens:#?}"); - let result = parser_lexer::expr.parse(tokens.as_slice()).unwrap(); + let tokens = parser_lexer::Tokens::new(&tokens); + let result = parser_lexer::expr.parse(tokens).unwrap(); println!(" {:#?}={}", result, result.eval()); } } diff --git a/examples/arithmetic/parser_lexer.rs b/examples/arithmetic/parser_lexer.rs index b78f6710..2277a099 100644 --- a/examples/arithmetic/parser_lexer.rs +++ b/examples/arithmetic/parser_lexer.rs @@ -13,7 +13,10 @@ use winnow::{ combinator::peek, combinator::repeat, combinator::{delimited, preceded, terminated}, + error::ContextError, + stream::TokenSlice, token::any, + token::literal, token::one_of, }; @@ -21,7 +24,8 @@ use winnow::{ #[allow(dead_code)] pub(crate) fn expr2(i: &mut &str) -> Result { let tokens = tokens.parse_next(i)?; - expr.parse_next(&mut tokens.as_slice()) + let mut tokens = Tokens::new(&tokens); + expr.parse_next(&mut tokens) } #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -32,6 +36,12 @@ pub enum Token { CloseParen, } +impl<'i> Parser, &'i Token, ContextError> for Token { + fn parse_next(&mut self, input: &mut Tokens<'i>) -> Result<&'i Token> { + literal(*self).parse_next(input).map(|t| &t[0]) + } +} + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum Oper { Add, @@ -40,31 +50,31 @@ pub enum Oper { Div, } -impl winnow::stream::ContainsToken for Token { +impl winnow::stream::ContainsToken<&'_ Token> for Token { #[inline(always)] - fn contains_token(&self, token: Token) -> bool { - *self == token + fn contains_token(&self, token: &'_ Token) -> bool { + self == token } } -impl winnow::stream::ContainsToken for &'_ [Token] { +impl winnow::stream::ContainsToken<&'_ Token> for &'_ [Token] { #[inline] - fn contains_token(&self, token: Token) -> bool { - self.iter().any(|t| *t == token) + fn contains_token(&self, token: &'_ Token) -> bool { + self.iter().any(|t| t == token) } } -impl winnow::stream::ContainsToken for &'_ [Token; LEN] { +impl winnow::stream::ContainsToken<&'_ Token> for &'_ [Token; LEN] { #[inline] - fn contains_token(&self, token: Token) -> bool { - self.iter().any(|t| *t == token) + fn contains_token(&self, token: &'_ Token) -> bool { + self.iter().any(|t| t == token) } } -impl winnow::stream::ContainsToken for [Token; LEN] { +impl winnow::stream::ContainsToken<&'_ Token> for [Token; LEN] { #[inline] - fn contains_token(&self, token: Token) -> bool { - self.iter().any(|t| *t == token) + fn contains_token(&self, token: &'_ Token) -> bool { + self.iter().any(|t| t == token) } } @@ -126,8 +136,10 @@ impl Display for Expr { } } +pub(crate) type Tokens<'i> = TokenSlice<'i, Token>; + /// Parse the tokens lexed in [`tokens`] -pub(crate) fn expr(i: &mut &[Token]) -> Result { +pub(crate) fn expr(i: &mut Tokens<'_>) -> Result { let init = term.parse_next(i)?; repeat( @@ -139,8 +151,8 @@ pub(crate) fn expr(i: &mut &[Token]) -> Result { ) .fold( move || init.clone(), - |acc, (op, val): (Token, Expr)| { - if op == Token::Oper(Oper::Add) { + |acc, (op, val): (&Token, Expr)| { + if *op == Token::Oper(Oper::Add) { Expr::Add(Box::new(acc), Box::new(val)) } else { Expr::Sub(Box::new(acc), Box::new(val)) @@ -150,7 +162,7 @@ pub(crate) fn expr(i: &mut &[Token]) -> Result { .parse_next(i) } -pub(crate) fn term(i: &mut &[Token]) -> Result { +pub(crate) fn term(i: &mut Tokens<'_>) -> Result { let init = factor.parse_next(i)?; repeat( @@ -162,8 +174,8 @@ pub(crate) fn term(i: &mut &[Token]) -> Result { ) .fold( move || init.clone(), - |acc, (op, val): (Token, Expr)| { - if op == Token::Oper(Oper::Mul) { + |acc, (op, val): (&Token, Expr)| { + if *op == Token::Oper(Oper::Mul) { Expr::Mul(Box::new(acc), Box::new(val)) } else { Expr::Div(Box::new(acc), Box::new(val)) @@ -173,10 +185,10 @@ pub(crate) fn term(i: &mut &[Token]) -> Result { .parse_next(i) } -pub(crate) fn factor(i: &mut &[Token]) -> Result { +pub(crate) fn factor(i: &mut Tokens<'_>) -> Result { alt(( - one_of(|t| matches!(t, Token::Value(_))).map(|t| match t { - Token::Value(v) => Expr::Value(v), + one_of(|t: &_| matches!(t, Token::Value(_))).map(|t: &_| match t { + Token::Value(v) => Expr::Value(*v), _ => unreachable!(), }), parens, @@ -184,8 +196,8 @@ pub(crate) fn factor(i: &mut &[Token]) -> Result { .parse_next(i) } -fn parens(i: &mut &[Token]) -> Result { - delimited(one_of(Token::OpenParen), expr, one_of(Token::CloseParen)) +fn parens(i: &mut Tokens<'_>) -> Result { + delimited(Token::OpenParen, expr, Token::CloseParen) .map(|e| Expr::Paren(Box::new(e))) .parse_next(i) } diff --git a/examples/arithmetic/test_parser_lexer.rs b/examples/arithmetic/test_parser_lexer.rs index cebecae8..87ee2b70 100644 --- a/examples/arithmetic/test_parser_lexer.rs +++ b/examples/arithmetic/test_parser_lexer.rs @@ -116,7 +116,14 @@ fn factor_test() { let expected = str![[r#" Ok( ( - [], + TokenSlice { + initial: [ + Value( + 3, + ), + ], + input: [], + }, Value( 3, ), @@ -125,13 +132,21 @@ Ok( "#]]; let input = tokens.parse(input).unwrap(); - assert_data_eq!(factor.parse_peek(&input).to_debug(), expected); + let input = Tokens::new(&input); + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); let input = " 12"; let expected = str![[r#" Ok( ( - [], + TokenSlice { + initial: [ + Value( + 12, + ), + ], + input: [], + }, Value( 12, ), @@ -140,13 +155,21 @@ Ok( "#]]; let input = tokens.parse(input).unwrap(); - assert_data_eq!(factor.parse_peek(&input).to_debug(), expected); + let input = Tokens::new(&input); + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); let input = "537 "; let expected = str![[r#" Ok( ( - [], + TokenSlice { + initial: [ + Value( + 537, + ), + ], + input: [], + }, Value( 537, ), @@ -155,13 +178,21 @@ Ok( "#]]; let input = tokens.parse(input).unwrap(); - assert_data_eq!(factor.parse_peek(&input).to_debug(), expected); + let input = Tokens::new(&input); + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); let input = " 24 "; let expected = str![[r#" Ok( ( - [], + TokenSlice { + initial: [ + Value( + 24, + ), + ], + input: [], + }, Value( 24, ), @@ -170,7 +201,8 @@ Ok( "#]]; let input = tokens.parse(input).unwrap(); - assert_data_eq!(factor.parse_peek(&input).to_debug(), expected); + let input = Tokens::new(&input); + assert_data_eq!(factor.parse_peek(input).to_debug(), expected); } #[test] @@ -179,7 +211,26 @@ fn term_test() { let expected = str![[r#" Ok( ( - [], + TokenSlice { + initial: [ + Value( + 12, + ), + Oper( + Mul, + ), + Value( + 2, + ), + Oper( + Div, + ), + Value( + 3, + ), + ], + input: [], + }, Div( Mul( Value( @@ -198,13 +249,33 @@ Ok( "#]]; let input = tokens.parse(input).unwrap(); - assert_data_eq!(term.parse_peek(&input).to_debug(), expected); + let input = Tokens::new(&input); + assert_data_eq!(term.parse_peek(input).to_debug(), expected); let input = " 12 *2 / 3"; let expected = str![[r#" Ok( ( - [], + TokenSlice { + initial: [ + Value( + 12, + ), + Oper( + Mul, + ), + Value( + 2, + ), + Oper( + Div, + ), + Value( + 3, + ), + ], + input: [], + }, Div( Mul( Value( @@ -223,13 +294,45 @@ Ok( "#]]; let input = tokens.parse(input).unwrap(); - assert_data_eq!(term.parse_peek(&input).to_debug(), expected); + let input = Tokens::new(&input); + assert_data_eq!(term.parse_peek(input).to_debug(), expected); let input = " 2* 3 *2 *2 / 3"; let expected = str![[r#" Ok( ( - [], + TokenSlice { + initial: [ + Value( + 2, + ), + Oper( + Mul, + ), + Value( + 3, + ), + Oper( + Mul, + ), + Value( + 2, + ), + Oper( + Mul, + ), + Value( + 2, + ), + Oper( + Div, + ), + Value( + 3, + ), + ], + input: [], + }, Div( Mul( Mul( @@ -258,13 +361,33 @@ Ok( "#]]; let input = tokens.parse(input).unwrap(); - assert_data_eq!(term.parse_peek(&input).to_debug(), expected); + let input = Tokens::new(&input); + assert_data_eq!(term.parse_peek(input).to_debug(), expected); let input = " 48 / 3/2"; let expected = str![[r#" Ok( ( - [], + TokenSlice { + initial: [ + Value( + 48, + ), + Oper( + Div, + ), + Value( + 3, + ), + Oper( + Div, + ), + Value( + 2, + ), + ], + input: [], + }, Div( Div( Value( @@ -283,7 +406,8 @@ Ok( "#]]; let input = tokens.parse(input).unwrap(); - assert_data_eq!(term.parse_peek(&input).to_debug(), expected); + let input = Tokens::new(&input); + assert_data_eq!(term.parse_peek(input).to_debug(), expected); } #[test] @@ -303,7 +427,8 @@ Ok( "#]]; let input = tokens.parse(input).unwrap(); - assert_data_eq!(expr.parse(&input).to_debug(), expected); + let input = Tokens::new(&input); + assert_data_eq!(expr.parse(input).to_debug(), expected); let input = " 12 + 6 - 4+ 3"; let expected = str![[r#" @@ -330,7 +455,8 @@ Ok( "#]]; let input = tokens.parse(input).unwrap(); - assert_data_eq!(expr.parse(&input).to_debug(), expected); + let input = Tokens::new(&input); + assert_data_eq!(expr.parse(input).to_debug(), expected); let input = " 1 + 2*3 + 4"; let expected = str![[r#" @@ -357,7 +483,8 @@ Ok( "#]]; let input = tokens.parse(input).unwrap(); - assert_data_eq!(expr.parse(&input).to_debug(), expected); + let input = Tokens::new(&input); + assert_data_eq!(expr.parse(input).to_debug(), expected); } #[test] @@ -374,7 +501,8 @@ Ok( "#]]; let input = tokens.parse(input).unwrap(); - assert_data_eq!(expr.parse(&input).to_debug(), expected); + let input = Tokens::new(&input); + assert_data_eq!(expr.parse(input).to_debug(), expected); let input = " 2* ( 3 + 4 ) "; let expected = str![[r#" @@ -398,7 +526,8 @@ Ok( "#]]; let input = tokens.parse(input).unwrap(); - assert_data_eq!(expr.parse(&input).to_debug(), expected); + let input = Tokens::new(&input); + assert_data_eq!(expr.parse(input).to_debug(), expected); let input = " 2*2 / ( 5 - 1) + 3"; let expected = str![[r#" @@ -432,5 +561,6 @@ Ok( "#]]; let input = tokens.parse(input).unwrap(); - assert_data_eq!(expr.parse(&input).to_debug(), expected); + let input = Tokens::new(&input); + assert_data_eq!(expr.parse(input).to_debug(), expected); } From ac8b11a11d9676a0adab0b307d170498ab4d6aa3 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 28 Feb 2025 10:18:57 -0600 Subject: [PATCH 07/11] docs(examples): Separate Token from TokenKind --- examples/arithmetic/parser_lexer.rs | 87 +++--- examples/arithmetic/test_parser_lexer.rs | 350 ++++++++++++++--------- 2 files changed, 264 insertions(+), 173 deletions(-) diff --git a/examples/arithmetic/parser_lexer.rs b/examples/arithmetic/parser_lexer.rs index 2277a099..8c787125 100644 --- a/examples/arithmetic/parser_lexer.rs +++ b/examples/arithmetic/parser_lexer.rs @@ -1,8 +1,6 @@ use std::fmt; use std::fmt::{Debug, Display, Formatter}; -use std::str::FromStr; - use winnow::prelude::*; use winnow::Result; use winnow::{ @@ -28,16 +26,28 @@ pub(crate) fn expr2(i: &mut &str) -> Result { expr.parse_next(&mut tokens) } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Token<'s> { + kind: TokenKind, + raw: &'s str, +} + +impl PartialEq for Token<'_> { + fn eq(&self, other: &TokenKind) -> bool { + self.kind == *other + } +} + #[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub enum Token { - Value(i64), +pub enum TokenKind { + Value, Oper(Oper), OpenParen, CloseParen, } -impl<'i> Parser, &'i Token, ContextError> for Token { - fn parse_next(&mut self, input: &mut Tokens<'i>) -> Result<&'i Token> { +impl<'i> Parser, &'i Token<'i>, ContextError> for TokenKind { + fn parse_next(&mut self, input: &mut Tokens<'i>) -> Result<&'i Token<'i>> { literal(*self).parse_next(input).map(|t| &t[0]) } } @@ -50,52 +60,54 @@ pub enum Oper { Div, } -impl winnow::stream::ContainsToken<&'_ Token> for Token { +impl winnow::stream::ContainsToken<&'_ Token<'_>> for TokenKind { #[inline(always)] - fn contains_token(&self, token: &'_ Token) -> bool { - self == token + fn contains_token(&self, token: &'_ Token<'_>) -> bool { + *self == token.kind } } -impl winnow::stream::ContainsToken<&'_ Token> for &'_ [Token] { +impl winnow::stream::ContainsToken<&'_ Token<'_>> for &'_ [TokenKind] { #[inline] - fn contains_token(&self, token: &'_ Token) -> bool { - self.iter().any(|t| t == token) + fn contains_token(&self, token: &'_ Token<'_>) -> bool { + self.iter().any(|t| *t == token.kind) } } -impl winnow::stream::ContainsToken<&'_ Token> for &'_ [Token; LEN] { +impl winnow::stream::ContainsToken<&'_ Token<'_>> for &'_ [TokenKind; LEN] { #[inline] - fn contains_token(&self, token: &'_ Token) -> bool { - self.iter().any(|t| t == token) + fn contains_token(&self, token: &'_ Token<'_>) -> bool { + self.iter().any(|t| *t == token.kind) } } -impl winnow::stream::ContainsToken<&'_ Token> for [Token; LEN] { +impl winnow::stream::ContainsToken<&'_ Token<'_>> for [TokenKind; LEN] { #[inline] - fn contains_token(&self, token: &'_ Token) -> bool { - self.iter().any(|t| t == token) + fn contains_token(&self, token: &'_ Token<'_>) -> bool { + self.iter().any(|t| *t == token.kind) } } /// Lex tokens /// /// See [`expr`] to parse the tokens -pub(crate) fn tokens(i: &mut &str) -> Result> { +pub(crate) fn tokens<'s>(i: &mut &'s str) -> Result>> { preceded(multispaces, repeat(1.., terminated(token, multispaces))).parse_next(i) } -fn token(i: &mut &str) -> Result { +fn token<'s>(i: &mut &'s str) -> Result> { dispatch! {peek(any); - '0'..='9' => digits.try_map(FromStr::from_str).map(Token::Value), - '(' => '('.value(Token::OpenParen), - ')' => ')'.value(Token::CloseParen), - '+' => '+'.value(Token::Oper(Oper::Add)), - '-' => '-'.value(Token::Oper(Oper::Sub)), - '*' => '*'.value(Token::Oper(Oper::Mul)), - '/' => '/'.value(Token::Oper(Oper::Div)), + '0'..='9' => digits.value(TokenKind::Value), + '(' => '('.value(TokenKind::OpenParen), + ')' => ')'.value(TokenKind::CloseParen), + '+' => '+'.value(TokenKind::Oper(Oper::Add)), + '-' => '-'.value(TokenKind::Oper(Oper::Sub)), + '*' => '*'.value(TokenKind::Oper(Oper::Mul)), + '/' => '/'.value(TokenKind::Oper(Oper::Div)), _ => fail, } + .with_taken() + .map(|(kind, raw)| Token { kind, raw }) .parse_next(i) } @@ -136,7 +148,7 @@ impl Display for Expr { } } -pub(crate) type Tokens<'i> = TokenSlice<'i, Token>; +pub(crate) type Tokens<'i> = TokenSlice<'i, Token<'i>>; /// Parse the tokens lexed in [`tokens`] pub(crate) fn expr(i: &mut Tokens<'_>) -> Result { @@ -145,14 +157,14 @@ pub(crate) fn expr(i: &mut Tokens<'_>) -> Result { repeat( 0.., ( - one_of([Token::Oper(Oper::Add), Token::Oper(Oper::Sub)]), + one_of([TokenKind::Oper(Oper::Add), TokenKind::Oper(Oper::Sub)]), term, ), ) .fold( move || init.clone(), - |acc, (op, val): (&Token, Expr)| { - if *op == Token::Oper(Oper::Add) { + |acc, (op, val): (&Token<'_>, Expr)| { + if op.kind == TokenKind::Oper(Oper::Add) { Expr::Add(Box::new(acc), Box::new(val)) } else { Expr::Sub(Box::new(acc), Box::new(val)) @@ -168,14 +180,14 @@ pub(crate) fn term(i: &mut Tokens<'_>) -> Result { repeat( 0.., ( - one_of([Token::Oper(Oper::Mul), Token::Oper(Oper::Div)]), + one_of([TokenKind::Oper(Oper::Mul), TokenKind::Oper(Oper::Div)]), factor, ), ) .fold( move || init.clone(), - |acc, (op, val): (&Token, Expr)| { - if *op == Token::Oper(Oper::Mul) { + |acc, (op, val): (&Token<'_>, Expr)| { + if op.kind == TokenKind::Oper(Oper::Mul) { Expr::Mul(Box::new(acc), Box::new(val)) } else { Expr::Div(Box::new(acc), Box::new(val)) @@ -187,17 +199,14 @@ pub(crate) fn term(i: &mut Tokens<'_>) -> Result { pub(crate) fn factor(i: &mut Tokens<'_>) -> Result { alt(( - one_of(|t: &_| matches!(t, Token::Value(_))).map(|t: &_| match t { - Token::Value(v) => Expr::Value(*v), - _ => unreachable!(), - }), + TokenKind::Value.try_map(|t: &Token<'_>| t.raw.parse::().map(Expr::Value)), parens, )) .parse_next(i) } fn parens(i: &mut Tokens<'_>) -> Result { - delimited(Token::OpenParen, expr, Token::CloseParen) + delimited(TokenKind::OpenParen, expr, TokenKind::CloseParen) .map(|e| Expr::Paren(Box::new(e))) .parse_next(i) } diff --git a/examples/arithmetic/test_parser_lexer.rs b/examples/arithmetic/test_parser_lexer.rs index 87ee2b70..cf411845 100644 --- a/examples/arithmetic/test_parser_lexer.rs +++ b/examples/arithmetic/test_parser_lexer.rs @@ -13,9 +13,10 @@ Ok( ( "", [ - Value( - 3, - ), + Token { + kind: Value, + raw: "3", + }, ], ), ) @@ -29,9 +30,10 @@ Ok( ( "", [ - Value( - 24, - ), + Token { + kind: Value, + raw: "24", + }, ], ), ) @@ -45,21 +47,30 @@ Ok( ( "", [ - Value( - 12, - ), - Oper( - Mul, - ), - Value( - 2, - ), - Oper( - Div, - ), - Value( - 3, - ), + Token { + kind: Value, + raw: "12", + }, + Token { + kind: Oper( + Mul, + ), + raw: "*", + }, + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Div, + ), + raw: "/", + }, + Token { + kind: Value, + raw: "3", + }, ], ), ) @@ -73,35 +84,58 @@ Ok( ( "", [ - Value( - 2, - ), - Oper( - Mul, - ), - Value( - 2, - ), - Oper( - Div, - ), - OpenParen, - Value( - 5, - ), - Oper( - Sub, - ), - Value( - 1, - ), - CloseParen, - Oper( - Add, - ), - Value( - 3, - ), + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Mul, + ), + raw: "*", + }, + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Div, + ), + raw: "/", + }, + Token { + kind: OpenParen, + raw: "(", + }, + Token { + kind: Value, + raw: "5", + }, + Token { + kind: Oper( + Sub, + ), + raw: "-", + }, + Token { + kind: Value, + raw: "1", + }, + Token { + kind: CloseParen, + raw: ")", + }, + Token { + kind: Oper( + Add, + ), + raw: "+", + }, + Token { + kind: Value, + raw: "3", + }, ], ), ) @@ -118,9 +152,10 @@ Ok( ( TokenSlice { initial: [ - Value( - 3, - ), + Token { + kind: Value, + raw: "3", + }, ], input: [], }, @@ -141,9 +176,10 @@ Ok( ( TokenSlice { initial: [ - Value( - 12, - ), + Token { + kind: Value, + raw: "12", + }, ], input: [], }, @@ -164,9 +200,10 @@ Ok( ( TokenSlice { initial: [ - Value( - 537, - ), + Token { + kind: Value, + raw: "537", + }, ], input: [], }, @@ -187,9 +224,10 @@ Ok( ( TokenSlice { initial: [ - Value( - 24, - ), + Token { + kind: Value, + raw: "24", + }, ], input: [], }, @@ -213,21 +251,30 @@ Ok( ( TokenSlice { initial: [ - Value( - 12, - ), - Oper( - Mul, - ), - Value( - 2, - ), - Oper( - Div, - ), - Value( - 3, - ), + Token { + kind: Value, + raw: "12", + }, + Token { + kind: Oper( + Mul, + ), + raw: "*", + }, + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Div, + ), + raw: "/", + }, + Token { + kind: Value, + raw: "3", + }, ], input: [], }, @@ -258,21 +305,30 @@ Ok( ( TokenSlice { initial: [ - Value( - 12, - ), - Oper( - Mul, - ), - Value( - 2, - ), - Oper( - Div, - ), - Value( - 3, - ), + Token { + kind: Value, + raw: "12", + }, + Token { + kind: Oper( + Mul, + ), + raw: "*", + }, + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Div, + ), + raw: "/", + }, + Token { + kind: Value, + raw: "3", + }, ], input: [], }, @@ -303,33 +359,50 @@ Ok( ( TokenSlice { initial: [ - Value( - 2, - ), - Oper( - Mul, - ), - Value( - 3, - ), - Oper( - Mul, - ), - Value( - 2, - ), - Oper( - Mul, - ), - Value( - 2, - ), - Oper( - Div, - ), - Value( - 3, - ), + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Mul, + ), + raw: "*", + }, + Token { + kind: Value, + raw: "3", + }, + Token { + kind: Oper( + Mul, + ), + raw: "*", + }, + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Mul, + ), + raw: "*", + }, + Token { + kind: Value, + raw: "2", + }, + Token { + kind: Oper( + Div, + ), + raw: "/", + }, + Token { + kind: Value, + raw: "3", + }, ], input: [], }, @@ -370,21 +443,30 @@ Ok( ( TokenSlice { initial: [ - Value( - 48, - ), - Oper( - Div, - ), - Value( - 3, - ), - Oper( - Div, - ), - Value( - 2, - ), + Token { + kind: Value, + raw: "48", + }, + Token { + kind: Oper( + Div, + ), + raw: "/", + }, + Token { + kind: Value, + raw: "3", + }, + Token { + kind: Oper( + Div, + ), + raw: "/", + }, + Token { + kind: Value, + raw: "2", + }, ], input: [], }, From 531704d216bf007fa98f78620171d7b66806240d Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 28 Feb 2025 10:31:12 -0600 Subject: [PATCH 08/11] docs(examples): Track Eof token This is inspired by rustc. This makes it easier to report errors with unexpected eof. --- examples/arithmetic/parser_lexer.rs | 23 ++++- examples/arithmetic/test_parser_lexer.rs | 104 +++++++++++++++++++++-- 2 files changed, 116 insertions(+), 11 deletions(-) diff --git a/examples/arithmetic/parser_lexer.rs b/examples/arithmetic/parser_lexer.rs index 8c787125..4641008a 100644 --- a/examples/arithmetic/parser_lexer.rs +++ b/examples/arithmetic/parser_lexer.rs @@ -7,7 +7,9 @@ use winnow::{ ascii::{digit1 as digits, multispace0 as multispaces}, combinator::alt, combinator::dispatch, + combinator::eof, combinator::fail, + combinator::opt, combinator::peek, combinator::repeat, combinator::{delimited, preceded, terminated}, @@ -44,6 +46,7 @@ pub enum TokenKind { Oper(Oper), OpenParen, CloseParen, + Eof, } impl<'i> Parser, &'i Token<'i>, ContextError> for TokenKind { @@ -92,7 +95,17 @@ impl winnow::stream::ContainsToken<&'_ Token<'_>> for [TokenKi /// /// See [`expr`] to parse the tokens pub(crate) fn tokens<'s>(i: &mut &'s str) -> Result>> { - preceded(multispaces, repeat(1.., terminated(token, multispaces))).parse_next(i) + let mut tokens: Vec<_> = + preceded(multispaces, repeat(1.., terminated(token, multispaces))).parse_next(i)?; + if let Some(eof) = opt(eof.map(|raw| Token { + kind: TokenKind::Eof, + raw, + })) + .parse_next(i)? + { + tokens.push(eof); + } + Ok(tokens) } fn token<'s>(i: &mut &'s str) -> Result> { @@ -154,7 +167,7 @@ pub(crate) type Tokens<'i> = TokenSlice<'i, Token<'i>>; pub(crate) fn expr(i: &mut Tokens<'_>) -> Result { let init = term.parse_next(i)?; - repeat( + let expr = repeat( 0.., ( one_of([TokenKind::Oper(Oper::Add), TokenKind::Oper(Oper::Sub)]), @@ -171,7 +184,11 @@ pub(crate) fn expr(i: &mut Tokens<'_>) -> Result { } }, ) - .parse_next(i) + .parse_next(i)?; + + opt(TokenKind::Eof).parse_next(i)?; + + Ok(expr) } pub(crate) fn term(i: &mut Tokens<'_>) -> Result { diff --git a/examples/arithmetic/test_parser_lexer.rs b/examples/arithmetic/test_parser_lexer.rs index cf411845..af08487e 100644 --- a/examples/arithmetic/test_parser_lexer.rs +++ b/examples/arithmetic/test_parser_lexer.rs @@ -17,6 +17,10 @@ Ok( kind: Value, raw: "3", }, + Token { + kind: Eof, + raw: "", + }, ], ), ) @@ -34,6 +38,10 @@ Ok( kind: Value, raw: "24", }, + Token { + kind: Eof, + raw: "", + }, ], ), ) @@ -71,6 +79,10 @@ Ok( kind: Value, raw: "3", }, + Token { + kind: Eof, + raw: "", + }, ], ), ) @@ -136,6 +148,10 @@ Ok( kind: Value, raw: "3", }, + Token { + kind: Eof, + raw: "", + }, ], ), ) @@ -156,8 +172,17 @@ Ok( kind: Value, raw: "3", }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, ], - input: [], }, Value( 3, @@ -180,8 +205,17 @@ Ok( kind: Value, raw: "12", }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, ], - input: [], }, Value( 12, @@ -204,8 +238,17 @@ Ok( kind: Value, raw: "537", }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, ], - input: [], }, Value( 537, @@ -228,8 +271,17 @@ Ok( kind: Value, raw: "24", }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, ], - input: [], }, Value( 24, @@ -275,8 +327,17 @@ Ok( kind: Value, raw: "3", }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, ], - input: [], }, Div( Mul( @@ -329,8 +390,17 @@ Ok( kind: Value, raw: "3", }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, ], - input: [], }, Div( Mul( @@ -403,8 +473,17 @@ Ok( kind: Value, raw: "3", }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, ], - input: [], }, Div( Mul( @@ -467,8 +546,17 @@ Ok( kind: Value, raw: "2", }, + Token { + kind: Eof, + raw: "", + }, + ], + input: [ + Token { + kind: Eof, + raw: "", + }, ], - input: [], }, Div( Div( From c9de301c622d5083cb165ef008debdef63b7b5c7 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 28 Feb 2025 10:38:21 -0600 Subject: [PATCH 09/11] docs(examples): Track Unknown tokens --- examples/arithmetic/parser_lexer.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/arithmetic/parser_lexer.rs b/examples/arithmetic/parser_lexer.rs index 4641008a..1625ff55 100644 --- a/examples/arithmetic/parser_lexer.rs +++ b/examples/arithmetic/parser_lexer.rs @@ -18,6 +18,7 @@ use winnow::{ token::any, token::literal, token::one_of, + token::take_till, }; /// Lex and parse @@ -46,6 +47,7 @@ pub enum TokenKind { Oper(Oper), OpenParen, CloseParen, + Unknown, Eof, } @@ -117,7 +119,8 @@ fn token<'s>(i: &mut &'s str) -> Result> { '-' => '-'.value(TokenKind::Oper(Oper::Sub)), '*' => '*'.value(TokenKind::Oper(Oper::Mul)), '/' => '/'.value(TokenKind::Oper(Oper::Div)), - _ => fail, + ' '| '\t'| '\r'| '\n' => fail, + _ => take_till(.., ('0'..='9', '(', ')', '+', '-', '*', '/')).value(TokenKind::Unknown), } .with_taken() .map(|(kind, raw)| Token { kind, raw }) From 1efa453622b6cbd3e94367ce0bcab58e845f6c48 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 28 Feb 2025 11:40:28 -0600 Subject: [PATCH 10/11] docs(stream): Point people in right direction --- src/stream/token.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/stream/token.rs b/src/stream/token.rs index e4f121b6..ccee13f5 100644 --- a/src/stream/token.rs +++ b/src/stream/token.rs @@ -15,6 +15,11 @@ use crate::stream::StreamIsPartial; use crate::stream::UpdateSlice; /// Specialized input for parsing lexed tokens +/// +/// Helpful impls +/// - Any `PartialEq` type (e.g. a `TokenKind` or `&str`) can be used with +/// [`literal`][crate::token::literal] +/// - A `PartialEq` for `&str` allows for using `&str` as a parser for tokens #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct TokenSlice<'t, T> { initial: &'t [T], From ab9830d84755dc75e44e3e190117719c7fa89116 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Fri, 28 Feb 2025 11:50:48 -0600 Subject: [PATCH 11/11] docs(topic): Split out a dedicated lexing section --- src/_topic/arithmetic.rs | 15 +-------------- src/_topic/lexing.rs | 23 +++++++++++++++++++++++ src/_topic/mod.rs | 2 ++ src/stream/token.rs | 2 ++ 4 files changed, 28 insertions(+), 14 deletions(-) create mode 100644 src/_topic/lexing.rs diff --git a/src/_topic/arithmetic.rs b/src/_topic/arithmetic.rs index d94b4fa4..dce37111 100644 --- a/src/_topic/arithmetic.rs +++ b/src/_topic/arithmetic.rs @@ -1,19 +1,6 @@ //! # Arithmetic //! -//! ## Direct evaluation -//! +//! This parses arithmetic expressions and directly evaluates them. //! ```rust #![doc = include_str!("../../examples/arithmetic/parser.rs")] //! ``` -//! -//! ## Parse to AST -//! -//! ```rust -#![doc = include_str!("../../examples/arithmetic/parser_ast.rs")] -//! ``` -//! -//! ## Parse to Tokens then AST -//! -//! ```rust -#![doc = include_str!("../../examples/arithmetic/parser_lexer.rs")] -//! ``` diff --git a/src/_topic/lexing.rs b/src/_topic/lexing.rs new file mode 100644 index 00000000..938453ae --- /dev/null +++ b/src/_topic/lexing.rs @@ -0,0 +1,23 @@ +//! # Lexing and Parsing +//! +//! ## Parse to AST +//! +//! The simplest way to write a parser is to parse directly to the AST. +//! +//! Example: +//! ```rust +#![doc = include_str!("../../examples/arithmetic/parser_ast.rs")] +//! ``` +//! +//! ## Lexing +//! +//! However, there are times when you may want to separate lexing from parsing. +//! Winnow provides [`TokenSlice`] to simplify this. +//! +//! Example: +//! ```rust +#![doc = include_str!("../../examples/arithmetic/parser_lexer.rs")] +//! ``` + +#![allow(unused_imports)] +use crate::stream::TokenSlice; diff --git a/src/_topic/mod.rs b/src/_topic/mod.rs index f279ac69..fcf83e0f 100644 --- a/src/_topic/mod.rs +++ b/src/_topic/mod.rs @@ -15,6 +15,7 @@ //! - [Implementing `FromStr`][fromstr] //! - [Performance][performance] //! - [Parsing Partial Input][partial] +//! - [Lexing and Parsing][lexing] //! - [Custom stream or token][stream] //! - [Custom errors][error] //! - [Debugging][crate::_tutorial::chapter_8] @@ -32,6 +33,7 @@ pub mod http; pub mod ini; pub mod json; pub mod language; +pub mod lexing; pub mod nom; pub mod partial; pub mod performance; diff --git a/src/stream/token.rs b/src/stream/token.rs index ccee13f5..97ff2e5f 100644 --- a/src/stream/token.rs +++ b/src/stream/token.rs @@ -20,6 +20,8 @@ use crate::stream::UpdateSlice; /// - Any `PartialEq` type (e.g. a `TokenKind` or `&str`) can be used with /// [`literal`][crate::token::literal] /// - A `PartialEq` for `&str` allows for using `&str` as a parser for tokens +/// +/// See also [Lexing and Parsing][crate::_topic::lexing]. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct TokenSlice<'t, T> { initial: &'t [T],