Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

docs(examples): Expand on lexing vs parsing #754

Merged
merged 11 commits into from
Feb 28, 2025
Merged

docs(examples): Expand on lexing vs parsing #754

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
fa1f22f
docs(examples): Pull tests out of arithmetic examples
epage Feb 28, 2025
e9bfd10
test(examples): Switch to snapbox
epage Feb 28, 2025
00787d6
test(examples): Don't assume full content on parsers
epage Feb 28, 2025
3d8aa27
docs(examples): Name the lexer for what it parses
epage Feb 22, 2025
77fc04f
docs(examples): Group lexer and parser logic
epage Feb 22, 2025
607c094
docs(examples): Use TokenSlice
epage Feb 22, 2025
ac8b11a
docs(examples): Separate Token from TokenKind
epage Feb 28, 2025
531704d
docs(examples): Track Eof token
epage Feb 28, 2025
c9de301
docs(examples): Track Unknown tokens
epage Feb 28, 2025
1efa453
docs(stream): Point people in right direction
epage Feb 28, 2025
ab9830d
docs(topic): Split out a dedicated lexing section
epage Feb 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions examples/arithmetic/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@ use winnow::prelude::*;
mod parser;
mod parser_ast;
mod parser_lexer;
#[cfg(test)]
mod test_parser;
#[cfg(test)]
mod test_parser_ast;
#[cfg(test)]
mod test_parser_lexer;

fn main() -> Result<(), lexopt::Error> {
let args = Args::parse()?;
Expand Down Expand Up @@ -31,9 +37,10 @@ fn calc(
println!(" {:#?}={}", result, result.eval());
}
Impl::Lexer => {
let tokens = parser_lexer::lex.parse(input)?;
let tokens = parser_lexer::tokens.parse(input)?;
println!(" {tokens:#?}");
let result = parser_lexer::expr.parse(tokens.as_slice()).unwrap();
let tokens = parser_lexer::Tokens::new(&tokens);
let result = parser_lexer::expr.parse(tokens).unwrap();
println!(" {:#?}={}", result, result.eval());
}
}
Expand Down
72 changes: 2 additions & 70 deletions examples/arithmetic/parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ pub(crate) fn expr(i: &mut &str) -> Result<i64> {
// We read an initial factor and for each time we find
// a * or / operator followed by another factor, we do
// the math by folding everything
fn term(i: &mut &str) -> Result<i64> {
pub(crate) fn term(i: &mut &str) -> Result<i64> {
let init = factor.parse_next(i)?;

repeat(0.., (one_of(['*', '/']), factor))
Expand All @@ -53,7 +53,7 @@ fn term(i: &mut &str) -> Result<i64> {
// We look for a digit suite, and try to convert it.
// If either str::from_utf8 or FromStr::from_str fail,
// we fallback to the parens parser defined above
fn factor(i: &mut &str) -> Result<i64> {
pub(crate) fn factor(i: &mut &str) -> Result<i64> {
delimited(
multispaces,
alt((digits.try_map(FromStr::from_str), parens)),
Expand All @@ -66,71 +66,3 @@ fn factor(i: &mut &str) -> Result<i64> {
fn parens(i: &mut &str) -> Result<i64> {
delimited('(', expr, ')').parse_next(i)
}

#[test]
fn factor_test() {
let input = "3";
let expected = Ok(("", 3));
assert_eq!(factor.parse_peek(input), expected);

let input = " 12";
let expected = Ok(("", 12));
assert_eq!(factor.parse_peek(input), expected);

let input = "537 ";
let expected = Ok(("", 537));
assert_eq!(factor.parse_peek(input), expected);

let input = " 24 ";
let expected = Ok(("", 24));
assert_eq!(factor.parse_peek(input), expected);
}

#[test]
fn term_test() {
let input = " 12 *2 / 3";
let expected = Ok(("", 8));
assert_eq!(term.parse_peek(input), expected);

let input = " 12 *2 / 3";
let expected = Ok(("", 8));
assert_eq!(term.parse_peek(input), expected);

let input = " 2* 3 *2 *2 / 3";
let expected = Ok(("", 8));
assert_eq!(term.parse_peek(input), expected);

let input = " 48 / 3/2";
let expected = Ok(("", 8));
assert_eq!(term.parse_peek(input), expected);
}

#[test]
fn expr_test() {
let input = " 1 + 2 ";
let expected = Ok(("", 3));
assert_eq!(expr.parse_peek(input), expected);

let input = " 12 + 6 - 4+ 3";
let expected = Ok(("", 17));
assert_eq!(expr.parse_peek(input), expected);

let input = " 1 + 2*3 + 4";
let expected = Ok(("", 11));
assert_eq!(expr.parse_peek(input), expected);
}

#[test]
fn parens_test() {
let input = " ( 2 )";
let expected = Ok(("", 2));
assert_eq!(expr.parse_peek(input), expected);

let input = " 2* ( 3 + 4 ) ";
let expected = Ok(("", 14));
assert_eq!(expr.parse_peek(input), expected);

let input = " 2*2 / ( 5 - 1) + 3";
let expected = Ok(("", 4));
assert_eq!(expr.parse_peek(input), expected);
}
87 changes: 2 additions & 85 deletions examples/arithmetic/parser_ast.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ pub(crate) fn expr(i: &mut &str) -> Result<Expr> {
.parse_next(i)
}

fn term(i: &mut &str) -> Result<Expr> {
pub(crate) fn term(i: &mut &str) -> Result<Expr> {
let init = factor.parse_next(i)?;

repeat(0.., (one_of(['*', '/']), factor))
Expand All @@ -84,7 +84,7 @@ fn term(i: &mut &str) -> Result<Expr> {
.parse_next(i)
}

fn factor(i: &mut &str) -> Result<Expr> {
pub(crate) fn factor(i: &mut &str) -> Result<Expr> {
delimited(
multispaces,
alt((digits.try_map(FromStr::from_str).map(Expr::Value), parens)),
Expand All @@ -98,86 +98,3 @@ fn parens(i: &mut &str) -> Result<Expr> {
.map(|e| Expr::Paren(Box::new(e)))
.parse_next(i)
}

#[test]
fn factor_test() {
let input = "3";
let expected = Ok(("", String::from("Value(3)")));
assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 12";
let expected = Ok(("", String::from("Value(12)")));
assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = "537 ";
let expected = Ok(("", String::from("Value(537)")));
assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 24 ";
let expected = Ok(("", String::from("Value(24)")));
assert_eq!(factor.map(|e| format!("{e:?}")).parse_peek(input), expected);
}

#[test]
fn term_test() {
let input = " 12 *2 / 3";
let expected = Ok(("", String::from("Div(Mul(Value(12), Value(2)), Value(3))")));
assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 12 *2 / 3";
let expected = Ok(("", String::from("Div(Mul(Value(12), Value(2)), Value(3))")));
assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 2* 3 *2 *2 / 3";
let expected = Ok((
"",
String::from("Div(Mul(Mul(Mul(Value(2), Value(3)), Value(2)), Value(2)), Value(3))"),
));
assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 48 / 3/2";
let expected = Ok(("", String::from("Div(Div(Value(48), Value(3)), Value(2))")));
assert_eq!(term.map(|e| format!("{e:?}")).parse_peek(input), expected);
}

#[test]
fn expr_test() {
let input = " 1 + 2 ";
let expected = Ok(("", String::from("Add(Value(1), Value(2))")));
assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 12 + 6 - 4+ 3";
let expected = Ok((
"",
String::from("Add(Sub(Add(Value(12), Value(6)), Value(4)), Value(3))"),
));
assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 1 + 2*3 + 4";
let expected = Ok((
"",
String::from("Add(Add(Value(1), Mul(Value(2), Value(3))), Value(4))"),
));
assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected);
}

#[test]
fn parens_test() {
let input = " ( 2 )";
let expected = Ok(("", String::from("Paren(Value(2))")));
assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 2* ( 3 + 4 ) ";
let expected = Ok((
"",
String::from("Mul(Value(2), Paren(Add(Value(3), Value(4))))"),
));
assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected);

let input = " 2*2 / ( 5 - 1) + 3";
let expected = Ok((
"",
String::from("Add(Div(Mul(Value(2), Value(2)), Paren(Sub(Value(5), Value(1)))), Value(3))"),
));
assert_eq!(expr.map(|e| format!("{e:?}")).parse_peek(input), expected);
}
Loading