Something went wrong. Try again.
Simple shell implementation.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819//! Tokenizes shell input and separates command arguments from output redirections.//!//! [`tokenize`] handles quotes, escapes, `>` / `1>` / `2>`, and `>>` / `1>>` / `2>>`.//! [`parse`] interprets those tokens without opening files or executing commands.//!//! This is a small shell syntax subset: expansion, pipelines, other file descriptors,//! and input redirection are not implemented. Unsupported shell syntax is//! not necessarily rejected; it may be interpreted as ordinary words.
/// Determines whether whitespace, quotes, and backslashes have special meaning.enum QuoteState { None, Single, Double,}
/// Invalid or incomplete syntax encountered during tokenization or parsing.#[derive(Debug, PartialEq, Eq, thiserror::Error)]pub enum ParseError { #[error("unclosed single quote")] UnclosedSingleQuote, #[error("unclosed double quote")] UnclosedDoubleQuote, #[error("trailing backslash")] TrailingBackslash, #[error("missing redirect target")] MissingRedirectTarget,}
/// A word with quoting/escaping processed, or a recognized shell operator.#[derive(Debug, PartialEq, Eq)]pub enum Token { /// May be empty when the input contains `''` or `""`. Word(String), /// Both `>` and `1>` redirect stdout and produce this token. RedirectOut, /// Both `>>` and `1>>` append to stdout's destination. AppendOut, /// '2>' produces this token. RedirectErr, /// `2>>` appends to stderr's destination. AppendErr,}
#[derive(Debug, PartialEq, Eq)]pub enum OutputStream { Stdout, Stderr,}
#[derive(Debug, PartialEq, Eq)]pub struct Redirection { pub stream: OutputStream, pub target: String, /// Append rather than truncate when opening the destination. pub append: bool,}
/// A simple command ready for the executor to resolve its output destinations.#[derive(Debug, Default, PartialEq, Eq)]pub struct ParsedCommand { /// Command name followed by its arguments, excluding redirection syntax. /// Empty for blank input or a command consisting only of redirections. pub args: Vec<String>, /// Destinations in source order, opened according to their append mode. /// The last destination for each stream receives that stream's output. pub redirects: Vec<Redirection>,}
/// Separates words and output redirections, which may appear anywhere in the input.////// For example, `echo >"my file" hello` yields arguments `["echo", "hello"]`/// and the output target `"my file"`. Empty input produces an empty command.////// # Errors////// Returns tokenization errors or [`ParseError::MissingRedirectTarget`] when a/// redirection is not followed by a word. A quoted empty filename is a valid word;/// any resulting file-open error is the executor's responsibility.pub fn parse(line: &str) -> Result<ParsedCommand, ParseError> { let mut tokens = tokenize(line)?.into_iter(); let mut command = ParsedCommand::default();
while let Some(token) = tokens.next() { match token { Token::Word(word) => command.args.push(word), Token::RedirectOut => match tokens.next() { Some(Token::Word(target)) => command.redirects.push(Redirection { stream: OutputStream::Stdout, append: false, target: target, }), _ => return Err(ParseError::MissingRedirectTarget), }, Token::AppendOut => match tokens.next() { Some(Token::Word(target)) => command.redirects.push(Redirection { stream: OutputStream::Stdout, target, append: true, }), _ => return Err(ParseError::MissingRedirectTarget), }, Token::AppendErr => match tokens.next() { Some(Token::Word(target)) => command.redirects.push(Redirection { stream: OutputStream::Stderr, target, append: true, }), _ => return Err(ParseError::MissingRedirectTarget), }, Token::RedirectErr => match tokens.next() { Some(Token::Word(target)) => command.redirects.push(Redirection { stream: OutputStream::Stderr, append: false, target: target, }), _ => return Err(ParseError::MissingRedirectTarget), }, } }
Ok(command)}
/// Splits input into words and output-redirection operators in one pass.////// Unquoted Unicode whitespace separates words. Quoted and unquoted fragments/// of a word are joined, and empty quoted words are preserved. Quoted or escaped/// `>` characters remain literal; operators do not require surrounding spaces./// Backslash-newline pairs are removed outside single quotes, but this function/// does not read additional input to complete a line yet.////// # Errors////// Returns an error for unclosed quotes or a trailing unquoted backslash./// Redirection targets are validated by [`parse`], not by this function.pub fn tokenize(line: &str) -> Result<Vec<Token>, ParseError> { let mut tokens = Vec::new(); let mut current_token = String::new(); let mut state = QuoteState::None; // An empty quoted word is still a token; String::is_empty cannot distinguish it // from whitespace between words. let mut token_started = false;
let mut characters = line.chars().peekable(); while let Some(character) = characters.next() { match (character, &state) { ('\\', QuoteState::None) => { let escaped = characters.next().ok_or(ParseError::TrailingBackslash)?; if escaped != '\n' { current_token.push(escaped); token_started = true; } } // Inside double quotes, only these characters consume the backslash. // peek leaves other characters for the next loop iteration. ('\\', QuoteState::Double) => match characters.peek() { Some('"' | '\\' | '$' | '`' | '\n') => { let escaped = characters.next().unwrap(); if escaped != '\n' { current_token.push(escaped); } } _ => current_token.push('\\'), }, ('\'', QuoteState::Single) => state = QuoteState::None, ('"', QuoteState::Double) => state = QuoteState::None, ('\'', QuoteState::None) => { state = QuoteState::Single; token_started = true; } ('"', QuoteState::None) => { state = QuoteState::Double; token_started = true; }
// `1>` explicitly selects stdout; the unquoted 1 must start a word. // In `hello1>out` or `"1">out`, the 1 belongs to the argument instead. ('1', QuoteState::None) if !token_started && characters.peek() == Some(&'>') => { characters.next(); if characters.next_if_eq(&'>').is_some() { tokens.push(Token::AppendOut); } else { tokens.push(Token::RedirectOut); } } ('2', QuoteState::None) if !token_started && characters.peek() == Some(&'>') => { characters.next(); if characters.next_if_eq(&'>').is_some() { tokens.push(Token::AppendErr); } else { tokens.push(Token::RedirectErr); } } // Finish an adjacent word before emitting the operator: hello>out. ('>', QuoteState::None) => { if token_started { tokens.push(Token::Word(std::mem::take(&mut current_token))); token_started = false; } if characters.next_if_eq(&'>').is_some() { tokens.push(Token::AppendOut); } else { tokens.push(Token::RedirectOut); } } (character, QuoteState::None) if character.is_whitespace() => { if token_started { tokens.push(Token::Word(std::mem::take(&mut current_token))); token_started = false; } } // Includes backslashes inside single quotes and nonmatching quote // characters, such as a single quote inside double quotes. (character, _) => { current_token.push(character); token_started = true; } } }
match state { QuoteState::Single => return Err(ParseError::UnclosedSingleQuote), QuoteState::Double => return Err(ParseError::UnclosedDoubleQuote), QuoteState::None => {} }
if token_started { tokens.push(Token::Word(current_token)); }
Ok(tokens)}
#[cfg(test)]mod tests { use crate::parser::OutputStream; use crate::parser::Redirection;
use super::{ParseError, Token, parse, tokenize};
#[test] fn parses_stderr_append_redirects() { for input in [ "echo hello 2>> err", "echo 2>>err hello", "2>>err echo hello", ] { let command = parse(input).unwrap(); assert_eq!(command.args, ["echo", "hello"]); assert_eq!( command.redirects, [Redirection { stream: OutputStream::Stderr, target: "err".into(), append: true, }] ); } assert_eq!( tokenize("2>>err").unwrap(), [Token::AppendErr, Token::Word("err".into())] ); }
#[test] fn preserves_literal_stderr_append_operators() { for input in ["echo '2>>'", r#"echo "2>>""#, r"echo 2\>\>"] { let command = parse(input).unwrap(); assert_eq!(command.args, ["echo", "2>>"]); assert!(command.redirects.is_empty()); } for (input, argument) in [ ("echo hello2>>out", "hello2"), ("echo 2 >>out", "2"), ("echo '2'>>out", "2"), (r"echo \2>>out", "2"), ] { let command = parse(input).unwrap(); assert_eq!(command.args, ["echo", argument]); assert_eq!( command.redirects, [Redirection { stream: OutputStream::Stdout, target: "out".into(), append: true, }] ); } }
#[test] fn parses_mixed_stderr_append_redirects_in_order() { let command = parse(r#"echo 2>>"first error" >out 2>last"#).unwrap(); assert_eq!( command.redirects, [ Redirection { stream: OutputStream::Stderr, target: "first error".into(), append: true }, Redirection { stream: OutputStream::Stdout, target: "out".into(), append: false }, Redirection { stream: OutputStream::Stderr, target: "last".into(), append: false }, ] ); }
#[test] fn rejects_missing_stderr_append_targets() { for input in ["echo 2>>", "echo 2>>>err", "echo 2>> >out"] { assert_eq!(parse(input), Err(ParseError::MissingRedirectTarget)); } }
#[test] fn parses_stdout_append_redirects() { for input in [ "echo hello>>out", "echo hello 1>>out", "1>>out echo hello", "echo >>out hello", ] { let command = parse(input).unwrap(); assert_eq!(command.args, ["echo", "hello"]); assert_eq!( command.redirects, [Redirection { stream: OutputStream::Stdout, target: "out".into(), append: true, }] ); } assert_eq!( tokenize("1>>out").unwrap(), [Token::AppendOut, Token::Word("out".into())] ); }
#[test] fn preserves_literal_append_operators() { for input in [r#"echo ">>""#, "echo '>>'", r"echo \>\>"] { let command = parse(input).unwrap(); assert_eq!(command.args, ["echo", ">>"]); assert!(command.redirects.is_empty()); } for (input, argument) in [ ("echo hello1>>out", "hello1"), ("echo '1'>>out", "1"), (r"echo \1>>out", "1"), ] { let command = parse(input).unwrap(); assert_eq!(command.args, ["echo", argument]); assert!(command.redirects[0].append); } }
#[test] fn preserves_mixed_append_and_truncate_order() { let command = parse(r#"echo >>"first file" >second 1>>third 2>errors"#).unwrap(); assert_eq!( command.redirects, [ Redirection { stream: OutputStream::Stdout, target: "first file".into(), append: true }, Redirection { stream: OutputStream::Stdout, target: "second".into(), append: false }, Redirection { stream: OutputStream::Stdout, target: "third".into(), append: true }, Redirection { stream: OutputStream::Stderr, target: "errors".into(), append: false }, ] ); }
#[test] fn parses_explicit_stdout_redirects() { for input in [ "echo Hello David 1> out", "echo Hello David 1>out", "1>out echo Hello David", ] { let command = parse(input).unwrap(); assert_eq!(command.args, ["echo", "Hello", "David"]); assert_eq!( command.redirects, [Redirection { stream: OutputStream::Stdout, append: false, target: "out".to_owned(), }] ); } assert_eq!(parse("echo 1>"), Err(ParseError::MissingRedirectTarget)); }
#[test] fn preserves_literal_ones_before_redirects() { for (input, argument) in [ ("echo hello1>out", "hello1"), ("echo 1 >out", "1"), ("echo '1'>out", "1"), (r#"echo "1">out"#, "1"), (r"echo \1>out", "1"), ("echo ''1>out", "1"), ] { let command = parse(input).unwrap(); assert_eq!(command.args, ["echo", argument]); assert_eq!( command.redirects, [Redirection { stream: OutputStream::Stdout, append: false, target: "out".into(), }] ); } }
#[test] fn preserves_quoted_and_escaped_explicit_redirects() { for input in ["echo '1>'", r#"echo "1>""#, r"echo 1\>"] { let command = parse(input).unwrap(); assert_eq!(command.args, ["echo", "1>"]); assert!(command.redirects.is_empty()); } }
#[test] fn tokenizes_redirects_without_spaces() { assert_eq!( tokenize("echo hello>out").unwrap(), [ Token::Word("echo".into()), Token::Word("hello".into()), Token::RedirectOut, Token::Word("out".into()) ] ); }
#[test] fn quoted_and_escaped_redirects_are_words() { assert_eq!( tokenize(r#"'>' ">" \>"#).unwrap(), [ Token::Word(">".into()), Token::Word(">".into()), Token::Word(">".into()) ] ); }
#[test] fn parses_redirects_anywhere_in_command() { for input in ["echo hello >out", "echo >out hello", ">out echo hello"] { let command = parse(input).unwrap(); assert_eq!(command.args, ["echo", "hello"]); assert_eq!( command.redirects, [Redirection { stream: OutputStream::Stdout, append: false, target: "out".into(), }] ); } }
#[test] fn parses_quoted_redirect_targets() { let command = parse(r#"echo >"my file" hello"#).unwrap(); assert_eq!(command.args, ["echo", "hello"]); assert_eq!( command.redirects, [Redirection { stream: OutputStream::Stdout, append: false, target: "my file".into(), }] ); }
#[test] fn preserves_redirect_order() { assert_eq!( parse("echo >first >second").unwrap().redirects, [ Redirection { stream: OutputStream::Stdout, append: false, target: "first".into() }, Redirection { stream: OutputStream::Stdout, append: false, target: "second".into() }, ] ); }
#[test] fn rejects_missing_redirect_targets() { for input in [ "echo >", "echo > >out", "echo >>", "echo 1>>", "echo >>>out", "echo >> >out", "echo 2>", "echo 2> >out", "echo > 2>err", ] { assert_eq!(parse(input), Err(ParseError::MissingRedirectTarget)); } }
#[test] fn parses_stderr_redirects() { for input in ["echo hello 2> err", "echo 2>err hello", "2>err echo hello"] { let command = parse(input).unwrap(); assert_eq!(command.args, ["echo", "hello"]); assert_eq!( command.redirects, [Redirection { stream: OutputStream::Stderr, append: false, target: "err".into(), }] ); } assert_eq!( tokenize("2>err").unwrap(), [Token::RedirectErr, Token::Word("err".into())] ); }
#[test] fn preserves_mixed_redirect_order() { let command = parse(r#"echo 2>"first error" hello >out 2>last"#).unwrap(); assert_eq!(command.args, ["echo", "hello"]); assert_eq!( command.redirects, [ Redirection { stream: OutputStream::Stderr, append: false, target: "first error".into() }, Redirection { stream: OutputStream::Stdout, append: false, target: "out".into() }, Redirection { stream: OutputStream::Stderr, append: false, target: "last".into() }, ] ); }
#[test] fn preserves_literal_twos_before_redirects() { for (input, argument) in [ ("echo hello2>out", "hello2"), ("echo 2 >out", "2"), ("echo '2'>out", "2"), (r#"echo "2">out"#, "2"), (r"echo \2>out", "2"), ("echo ''2>out", "2"), ] { let command = parse(input).unwrap(); assert_eq!(command.args, ["echo", argument]); assert_eq!( command.redirects, [Redirection { stream: OutputStream::Stdout, append: false, target: "out".into(), }] ); } }
#[test] fn preserves_quoted_and_escaped_stderr_operators() { for input in ["echo '2>'", r#"echo "2>""#, r"echo 2\>"] { let command = parse(input).unwrap(); assert_eq!(command.args, ["echo", "2>"]); assert!(command.redirects.is_empty()); } }
#[test] fn escapes_characters_outside_quotes() { assert_eq!( parse(r#"echo hello\ world \'quoted\' \"double\" \\ \a"#) .unwrap() .args, ["echo", "hello world", "'quoted'", "\"double\"", "\\", "a"] ); }
#[test] fn preserves_backslashes_inside_single_quotes() { assert_eq!( parse(r#"echo 'hello\ world\\'"#).unwrap().args, ["echo", r"hello\ world\\"] ); assert_eq!( parse(r"echo '\' next").unwrap().args, ["echo", "\\", "next"] ); }
#[test] fn escapes_special_characters_inside_double_quotes() { assert_eq!( parse(r#"echo "say \"hello\" \\ \$ \`""#).unwrap().args, ["echo", "say \"hello\" \\ $ `"] ); }
#[test] fn preserves_other_backslashes_inside_double_quotes() { assert_eq!( parse(r#"echo "hello\ world\n\'""#).unwrap().args, ["echo", r"hello\ world\n\'"] ); }
#[test] fn removes_escaped_newlines_outside_single_quotes() { assert_eq!(parse("echo hel\\\nlo").unwrap().args, ["echo", "hello"]); assert_eq!(parse("echo \"hel\\\nlo\"").unwrap().args, ["echo", "hello"]); assert!(parse("\\\n").unwrap().args.is_empty()); assert_eq!( parse("echo 'hel\\\nlo'").unwrap().args, ["echo", "hel\\\nlo"] ); }
#[test] fn rejects_trailing_backslash() { assert_eq!(parse("echo hello\\"), Err(ParseError::TrailingBackslash)); }
#[test] fn escaped_quote_does_not_close_double_quotes() { assert_eq!( parse(r#"echo "hello\""#), Err(ParseError::UnclosedDoubleQuote) ); assert_eq!( parse("echo \"hello\\"), Err(ParseError::UnclosedDoubleQuote) ); }
#[test] fn rejects_unclosed_single_quotes() { for input in ["'", "echo 'hello", "echo 'hello\""] { assert_eq!(parse(input), Err(ParseError::UnclosedSingleQuote)); } }
#[test] fn rejects_unclosed_double_quotes() { for input in ["\"", "echo \"hello", "echo \"hello'"] { assert_eq!(parse(input), Err(ParseError::UnclosedDoubleQuote)); } }
#[test] fn tokenizes_a_single_word() { assert_eq!(parse("echo").unwrap().args, ["echo"]); }
#[test] fn tokenizes_multiple_words() { assert_eq!( parse("echo hello world").unwrap().args, ["echo", "hello", "world"] ); }
#[test] fn ignores_extra_whitespace() { assert_eq!(parse(" echo hello ").unwrap().args, ["echo", "hello"]); }
#[test] fn keeps_single_quoted_text_in_one_token() { assert_eq!( parse("echo 'hello world'").unwrap().args, ["echo", "hello world"] ); }
#[test] fn keeps_double_quoted_text_in_one_token() { assert_eq!( parse(r#"echo "hello world""#).unwrap().args, ["echo", "hello world"] ); }
#[test] fn preserves_whitespace_inside_double_quotes() { assert_eq!( parse("echo \" hello\t world \"").unwrap().args, ["echo", " hello\t world "] ); }
#[test] fn separates_arguments_after_double_quotes() { assert_eq!( parse(r#"echo "hello world" next"#).unwrap().args, ["echo", "hello world", "next"] ); }
#[test] fn combines_double_quoted_and_unquoted_parts() { assert_eq!( parse(r#"echo before" middle "after"#).unwrap().args, ["echo", "before middle after"] ); }
#[test] fn preserves_single_quotes_inside_double_quotes() { assert_eq!( parse(r#"echo "it's 'quoted'""#).unwrap().args, ["echo", "it's 'quoted'"] ); }
#[test] fn preserves_double_quotes_inside_single_quotes() { assert_eq!( parse(r#"echo 'say "hello"'"#).unwrap().args, ["echo", "say \"hello\""] ); }
#[test] fn combines_adjacent_single_and_double_quoted_parts() { assert_eq!( parse(r#"echo 'hello '"world""#).unwrap().args, ["echo", "hello world"] ); }
#[test] fn preserves_empty_double_quoted_arguments() { assert_eq!( parse(r#"echo "" next """#).unwrap().args, ["echo", "", "next", ""] ); }
#[test] fn preserves_empty_single_quoted_arguments() { assert_eq!( parse("echo '' next ''").unwrap().args, ["echo", "", "next", ""] ); }
#[test] fn ignores_empty_and_whitespace_only_input() { assert!(parse("").unwrap().args.is_empty()); assert!(parse(" \t\n ").unwrap().args.is_empty()); }
#[test] fn combines_adjacent_empty_quotes_into_one_token() { assert_eq!( parse(r#" ''"" ""hello'' "#).unwrap().args, ["", "hello"] ); }
#[test] fn combines_quoted_and_unquoted_parts_of_a_token() { assert_eq!( parse("echo hello' world'").unwrap().args, ["echo", "hello world"] ); }}