Skip to content

Commit

Permalink
Merge pull request #754 from epage/lex
Browse files Browse the repository at this point in the history 
docs(examples): Expand on lexing vs parsing
  • Loading branch information
@epage
epage authored Feb 28, 2025
2 parents 70e1f43 + ab9830d commit a6e7c9b
Show file tree
Hide file tree
Showing 11 changed files with 1,411 additions and 366 deletions.
11 changes: 9 additions & 2 deletions examples/arithmetic/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@ use winnow::prelude::*;
mod parser;
mod parser_ast;
mod parser_lexer;
#[cfg(test)]
mod test_parser;
#[cfg(test)]
mod test_parser_ast;
#[cfg(test)]
mod test_parser_lexer;

fn main() -> Result<(), lexopt::Error> {
let args = Args::parse()?;
Expand Down Expand Up @@ -31,9 +37,10 @@ fn calc(
println!(" {:#?}={}", result, result.eval());
}
Impl::Lexer => {
let tokens = parser_lexer::lex.parse(input)?;
let tokens = parser_lexer::tokens.parse(input)?;
println!(" {tokens:#?}");
let result = parser_lexer::expr.parse(tokens.as_slice()).unwrap();
let tokens = parser_lexer::Tokens::new(&tokens);
let result = parser_lexer::expr.parse(tokens).unwrap();
println!(" {:#?}={}", result, result.eval());
}
}
Expand Down
72 changes: 2 additions & 70 deletions examples/arithmetic/parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ pub(crate) fn expr(i: &mut &str) -> Result<i64> {
// We read an initial factor and for each time we find
// a * or / operator followed by another factor, we do
// the math by folding everything
fn term(i: &mut &str) -> Result<i64> {
pub(crate) fn term(i: &mut &str) -> Result<i64> {
let init = factor.parse_next(i)?;

repeat(0.., (one_of(['*', '/']), factor))
Expand All @@ -53,7 +53,7 @@ fn term(i: &mut &str) -> Result<i64> {
// We look for a digit suite, and try to convert it.
// If either str::from_utf8 or FromStr::from_str fail,
// we fallback to the parens parser defined above
fn factor(i: &mut &str) -> Result<i64> {
pub(crate) fn factor(i: &mut &str) -> Result<i64> {
delimited(
multispaces,
alt((digits.try_map(FromStr::from_str), parens)),
Expand All @@ -66,71 +66,3 @@ fn factor(i: &mut &str) -> Result<i64> {
fn parens(i: &mut &str) -> Result<i64> {
delimited('(', expr, ')').parse_next(i)
}

#[test]
fn factor_test() {
let input = "3";
let expected = Ok(("", 3));
assert_eq!(factor.parse_peek(input), expected);

let input = " 12";
let expected = Ok(("", 12));
assert_eq!(factor.parse_peek(input), expected);

let input = "537 ";
let expected = Ok(("", 537));
assert_eq!(factor.parse_peek(input), expected);

let input = " 24 ";
let expected = Ok(("", 24));
assert_eq!(factor.parse_peek(input), expected);
}

#[test]
fn term_test() {
let input = " 12 *2 / 3";
let expected = Ok(("", 8));
assert_eq!(term.parse_peek(input), expected);

let input = " 12 *2 / 3";
let expected = Ok(("", 8));
assert_eq!(term.parse_peek(input), expected);

let input = " 2* 3 *2 *2 / 3";
let expected = Ok(("", 8));
assert_eq!(term.parse_peek(input), expected);

let input = " 48 / 3/2";
let expected = Ok(("", 8));
assert_eq!(term.parse_peek(input), expected);
}

#[test]
fn expr_test() {
let input = " 1 + 2 ";
let expected = Ok(("", 3));
assert_eq!(expr.parse_peek(input), expected);

let input = " 12 + 6 - 4+ 3";
let expected = Ok(("", 17));
assert_eq!(expr.parse_peek(input), expected);

let input = " 1 + 2*3 + 4";
let expected = Ok(("", 11));
assert_eq!(expr.parse_peek(input), expected);
}

#[test]
fn parens_test() {
let input = " ( 2 )";
let expected = Ok(("", 2));
assert_eq!(expr.parse_peek(input), expected);

let input = " 2* ( 3 + 4 ) ";
let expected = Ok(("", 14));
assert_eq!(expr.parse_peek(input), expected);

let input = " 2*2 / ( 5 - 1) + 3";
let expected = Ok(("", 4));
assert_eq!(expr.parse_peek(input), expected);
}
87 changes: 2 additions & 85 deletions examples/arithmetic/parser_ast.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ pub(crate) fn expr(i: &mut &str) -> Result<Expr> {
.parse_next(i)
}

fn term(i: &mut &str) -> Result<Expr> {
pub(crate) fn term(i: &mut &str) -> Result<Expr> {
let init = factor.parse_next(i)?;

repeat(0.., (one_of(['*', '/']), factor))
Expand All @@ -84,7 +84,7 @@ fn term(i: &mut &str) -> Result<Expr> {
.parse_next(i)
}

fn factor(i: &mut &str) -> Result<Expr> {
pub(crate) fn factor(i: &mut &str) -> Result<Expr> {
delimited(
multispaces,
alt((digits.try_map(FromStr::from_str).map(Expr::Value), parens)),
Expand All @@ -98,86 +98,3 @@ fn parens(i: &mut &str) -> Result<Expr> {
.map(|e| Expr::Paren(Box::new(e)))
.parse_next(i)
}

#[test]
fn factor_test() {
let input = "3";
let expected = Ok(("", String::from("Value(3)")));
assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 12";
let expected = Ok(("", String::from("Value(12)")));
assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = "537 ";
let expected = Ok(("", String::from("Value(537)")));
assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 24 ";
let expected = Ok(("", String::from("Value(24)")));
assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected);
}

#[test]
fn term_test() {
let input = " 12 *2 / 3";
let expected = Ok(("", String::from("Div(Mul(Value(12), Value(2)), Value(3))")));
assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 12 *2 / 3";
let expected = Ok(("", String::from("Div(Mul(Value(12), Value(2)), Value(3))")));
assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 2* 3 *2 *2 / 3";
let expected = Ok((
"",
String::from("Div(Mul(Mul(Mul(Value(2), Value(3)), Value(2)), Value(2)), Value(3))"),
));
assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 48 / 3/2";
let expected = Ok(("", String::from("Div(Div(Value(48), Value(3)), Value(2))")));
assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected);
}

#[test]
fn expr_test() {
let input = " 1 + 2 ";
let expected = Ok(("", String::from("Add(Value(1), Value(2))")));
assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 12 + 6 - 4+ 3";
let expected = Ok((
"",
String::from("Add(Sub(Add(Value(12), Value(6)), Value(4)), Value(3))"),
));
assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 1 + 2*3 + 4";
let expected = Ok((
"",
String::from("Add(Add(Value(1), Mul(Value(2), Value(3))), Value(4))"),
));
assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected);
}

#[test]
fn parens_test() {
let input = " ( 2 )";
let expected = Ok(("", String::from("Paren(Value(2))")));
assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 2* ( 3 + 4 ) ";
let expected = Ok((
"",
String::from("Mul(Value(2), Paren(Add(Value(3), Value(4))))"),
));
assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 2*2 / ( 5 - 1) + 3";
let expected = Ok((
"",
String::from("Add(Div(Mul(Value(2), Value(2)), Paren(Sub(Value(5), Value(1)))), Value(3))"),
));
assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected);
}
Loading

0 comments on commit a6e7c9b

Please sign in to comment.