ast: make Ast generic

We can parse two different things via Ast:

1. A regular job list
2. A freestanding argument list, as used in `complete --arguments ...`

This second case is specific to one use.

Prior to this commit, we parsed the Ast and then "forgot" what we parsed,
storing a &dyn Node. Then we had to cast it to the right type, and assert,
and etc.

Make Ast generic over the Node type it parsed, and default the Node type to
JobList. This simplifies call sites.
This commit is contained in:
Peter Ammon
2025-05-01 21:59:45 -07:00
parent ccc75d08f3
commit ccfe949514
13 changed files with 93 additions and 124 deletions

View File

@@ -1698,43 +1698,63 @@ pub struct Extras {
pub errors: SourceRangeList,
}
/// Parse a job list.
pub fn parse(src: &wstr, flags: ParseTreeFlags, out_errors: Option<&mut ParseErrorList>) -> Ast {
let mut pops = Populator::new(src, flags, Type::job_list, out_errors);
let mut list = JobList::default();
pops.populate_list(&mut list, true);
finalize_parse(pops, list)
}
/// Parse a FreestandingArgumentList.
pub fn parse_argument_list(
src: &wstr,
flags: ParseTreeFlags,
out_errors: Option<&mut ParseErrorList>,
) -> Ast<FreestandingArgumentList> {
let mut pops = Populator::new(src, flags, Type::freestanding_argument_list, out_errors);
let mut list = FreestandingArgumentList::default();
pops.populate_list(&mut list.arguments, true);
finalize_parse(pops, list)
}
// Given that we have populated some top node, add all the extras that we want and produce an Ast.
fn finalize_parse<N: Node>(mut pops: Populator<'_>, top: N) -> Ast<N> {
// Chomp trailing extras, etc.
pops.chomp_extras(Type::job_list);
let any_error = pops.any_error;
let extras = Extras {
comments: pops.tokens.comment_ranges,
semis: pops.semis,
errors: pops.errors,
};
Ast {
top,
any_error,
extras,
}
}
/// The ast type itself.
pub struct Ast {
pub struct Ast<N: Node = JobList> {
// The top node.
// Its type depends on what was requested to parse.
top: Box<dyn Node>,
top: N,
/// Whether any errors were encountered during parsing.
any_error: bool,
/// Extra fields.
pub extras: Extras,
}
impl Ast {
/// Construct an ast by parsing `src` as a job list.
/// The ast attempts to produce `type` as the result.
/// `type` may only be JobList or FreestandingArgumentList.
pub fn parse(
src: &wstr,
flags: ParseTreeFlags,
out_errors: Option<&mut ParseErrorList>,
) -> Self {
parse_from_top(src, flags, out_errors, Type::job_list)
}
/// Like parse(), but constructs a freestanding_argument_list.
pub fn parse_argument_list(
src: &wstr,
flags: ParseTreeFlags,
out_errors: Option<&mut ParseErrorList>,
) -> Self {
parse_from_top(src, flags, out_errors, Type::freestanding_argument_list)
}
impl<N: Node> Ast<N> {
/// Return a traversal, allowing iteration over the nodes.
pub fn walk(&'_ self) -> Traversal<'_> {
Traversal::new(self.top.as_node())
Traversal::new(&self.top)
}
/// Return the top node. This has the type requested in the 'parse' method.
pub fn top(&self) -> &dyn Node {
&*self.top
pub fn top(&self) -> &N {
&self.top
}
/// Return whether any errors were encountered during parsing.
pub fn errored(&self) -> bool {
@@ -3081,44 +3101,6 @@ enum ParserStatus {
unwinding,
}
fn parse_from_top(
src: &wstr,
flags: ParseTreeFlags,
out_errors: Option<&mut ParseErrorList>,
top_type: Type,
) -> Ast {
let mut pops = Populator::new(src, flags, top_type, out_errors);
let top: Box<dyn Node> = match top_type {
Type::job_list => {
let mut list: Box<JobList> = Box::default();
pops.populate_list(&mut *list, true);
list
}
Type::freestanding_argument_list => {
let mut list = Box::<FreestandingArgumentList>::default();
pops.populate_list(&mut list.arguments, true);
list
}
_ => panic!("Invalid top type"),
};
// Chomp trailing extras, etc.
pops.chomp_extras(Type::job_list);
let any_error = pops.any_error;
let extras = Extras {
comments: pops.tokens.comment_ranges,
semis: pops.semis,
errors: pops.errors,
};
Ast {
top,
any_error,
extras,
}
}
/// Return tokenizer flags corresponding to parse tree flags.
impl From<ParseTreeFlags> for TokFlags {
fn from(flags: ParseTreeFlags) -> Self {
@@ -3207,7 +3189,7 @@ fn keyword_for_token(tok: TokenType, token: &wstr) -> ParseKeyword {
fn test_ast_parse() {
let _cleanup = test_init();
let src = L!("echo");
let ast = Ast::parse(src, ParseTreeFlags::empty(), None);
let ast = parse(src, ParseTreeFlags::empty(), None);
assert!(!ast.any_error);
}

View File

@@ -22,7 +22,7 @@
#![allow(clippy::uninlined_format_args)]
use fish::{
ast::Ast,
ast,
builtins::{
fish_indent, fish_key_reader,
shared::{
@@ -226,7 +226,7 @@ fn run_command_list(parser: &Parser, cmds: &[OsString]) -> Result<(), libc::c_in
let cmd_wcs = str2wcstring(cmd.as_bytes());
let mut errors = ParseErrorList::new();
let ast = Ast::parse(&cmd_wcs, ParseTreeFlags::empty(), Some(&mut errors));
let ast = ast::parse(&cmd_wcs, ParseTreeFlags::empty(), Some(&mut errors));
let errored = ast.errored() || {
parse_util_detect_errors_in_ast(&ast, &cmd_wcs, Some(&mut errors)).is_err()
};

View File

@@ -1,5 +1,5 @@
use super::prelude::*;
use crate::ast::{Ast, Kind, Leaf};
use crate::ast::{self, Kind, Leaf};
use crate::common::{unescape_string, UnescapeFlags, UnescapeStringStyle};
use crate::complete::Completion;
use crate::expand::{expand_string, ExpandFlags, ExpandResultCode};
@@ -108,7 +108,7 @@ fn strip_dollar_prefixes(insert_mode: AppendMode, prefix: &wstr, insert: &wstr)
}
insert.find(L!("$ "))?; // Early return.
let source = prefix.to_owned() + insert;
let ast = Ast::parse(
let ast = ast::parse(
&source,
ParseTreeFlags::ACCEPT_INCOMPLETE_TOKENS | ParseTreeFlags::LEAVE_UNTERMINATED,
None,

View File

@@ -816,7 +816,6 @@ fn visit_begin_header(&mut self, node: &ast::BeginHeader) {
// Prettify our ast traversal, populating the output.
fn prettify_traversal(&mut self) {
use ast::Kind;
while let Some(node) = self.traversal.next() {
// Leaf nodes we just visit their text.
if node.as_keyword().is_some() {
@@ -1242,7 +1241,7 @@ struct TokenRange {
// Entry point for prettification.
fn prettify(streams: &mut IoStreams, src: &wstr, do_indent: bool) -> WString {
if DUMP_PARSE_TREE.load() {
let ast = Ast::parse(
let ast = ast::parse(
src,
ParseTreeFlags::LEAVE_UNTERMINATED
| ParseTreeFlags::INCLUDE_COMMENTS
@@ -1257,7 +1256,7 @@ fn prettify(streams: &mut IoStreams, src: &wstr, do_indent: bool) -> WString {
metrics.visit(ast.top());
streams.err.appendln(format!("{}", metrics));
}
let ast = Ast::parse(src, parse_flags(), None);
let ast = ast::parse(src, parse_flags(), None);
let mut printer = PrettyPrinter::new(src, &ast, do_indent);
printer.prettify()
}

View File

@@ -1,7 +1,7 @@
//! Functions for syntax highlighting.
use crate::abbrs::{self, with_abbrs};
use crate::ast::{
self, Argument, Ast, BlockStatement, BlockStatementHeader, BraceStatement, DecoratedStatement,
self, Argument, BlockStatement, BlockStatementHeader, BraceStatement, DecoratedStatement,
Keyword, Kind, Node, NodeVisitor, Redirection, Token, VariableAssignment,
};
use crate::builtins::shared::builtin_exists;
@@ -271,16 +271,14 @@ fn autosuggest_parse_command(
buff: &wstr,
ctx: &OperationContext<'_>,
) -> Option<(WString, WString)> {
let ast = Ast::parse(
let ast = ast::parse(
buff,
ParseTreeFlags::CONTINUE_AFTER_ERROR | ParseTreeFlags::ACCEPT_INCOMPLETE_TOKENS,
None,
);
// Find the first statement.
let Kind::JobList(job_list) = ast.top().kind() else {
panic!("Expected job list");
};
let job_list: &ast::JobList = ast.top();
let jc = job_list.get(0)?;
let first_statement = jc.job.statement.as_decorated_statement()?;
@@ -711,7 +709,7 @@ pub fn highlight(&mut self) -> ColorArray {
| ParseTreeFlags::ACCEPT_INCOMPLETE_TOKENS
| ParseTreeFlags::LEAVE_UNTERMINATED
| ParseTreeFlags::SHOW_EXTRA_SEMIS;
let ast = Ast::parse(self.buff, ast_flags, None);
let ast = ast::parse(self.buff, ast_flags, None);
self.visit_children(ast.top());
if self.ctx.check_cancel() {

View File

@@ -47,7 +47,7 @@
use rand::Rng;
use crate::{
ast::{Ast, Kind, Node},
ast::{self, Kind, Node},
common::{
str2wcstring, unescape_string, valid_var_name, wcs2zstring, CancelChecker,
UnescapeStringStyle,
@@ -1496,7 +1496,7 @@ fn should_import_bash_history_line(line: &wstr) -> bool {
}
}
if Ast::parse(line, ParseTreeFlags::empty(), None).errored() {
if ast::parse(line, ParseTreeFlags::empty(), None).errored() {
return false;
}
@@ -1581,7 +1581,7 @@ pub fn add_pending_with_file_detection(
// Find all arguments that look like they could be file paths.
let mut needs_sync_write = false;
let ast = Ast::parse(s, ParseTreeFlags::empty(), None);
let ast = ast::parse(s, ParseTreeFlags::empty(), None);
let mut potential_paths = Vec::new();
for node in ast.walk() {

View File

@@ -4,7 +4,7 @@
use std::pin::Pin;
use std::sync::Arc;
use crate::ast::{Ast, Node};
use crate::ast::{self, Ast, Node};
use crate::common::{assert_send, assert_sync};
use crate::parse_constants::{
token_type_user_presentable_description, ParseErrorCode, ParseErrorList, ParseKeyword,
@@ -187,7 +187,7 @@ pub fn parse_source(
flags: ParseTreeFlags,
errors: Option<&mut ParseErrorList>,
) -> Option<ParsedSourceRef> {
let ast = Ast::parse(&src, flags, errors);
let ast = ast::parse(&src, flags, errors);
if ast.errored() && !flags.contains(ParseTreeFlags::CONTINUE_AFTER_ERROR) {
None
} else {

View File

@@ -767,7 +767,7 @@ fn compute_indents(src: &wstr, initial_indent: i32) -> Vec<i32> {
// the last node we visited becomes the input indent of the next. I.e. in the case of 'switch
// foo ; cas', we get an invalid parse tree (since 'cas' is not valid) but we indent it as if it
// were a case item list.
let ast = Ast::parse(
let ast = ast::parse(
src,
ParseTreeFlags::CONTINUE_AFTER_ERROR
| ParseTreeFlags::INCLUDE_COMMENTS
@@ -996,7 +996,6 @@ impl<'a> NodeVisitor<'a> for IndentVisitor<'a> {
// Default implementation is to just visit children.
fn visit(&mut self, node: &'a dyn Node) {
let mut inc_dec = (0, 0);
use ast::Kind;
match node.kind() {
Kind::JobList(_) | Kind::AndorJobList(_) => {
// Job lists are never unwound.
@@ -1133,7 +1132,7 @@ pub fn parse_util_detect_errors(
// Parse the input string into an ast. Some errors are detected here.
let mut parse_errors = ParseErrorList::new();
let ast = Ast::parse(buff_src, parse_flags, Some(&mut parse_errors));
let ast = ast::parse(buff_src, parse_flags, Some(&mut parse_errors));
if allow_incomplete {
// Issue #1238: If the only error was unterminated quote, then consider this to have parsed
// successfully.
@@ -1328,14 +1327,14 @@ pub fn parse_util_detect_errors_in_argument_list(
// Parse the string as a freestanding argument list.
let mut errors = ParseErrorList::new();
let ast = Ast::parse_argument_list(arg_list_src, ParseTreeFlags::empty(), Some(&mut errors));
let ast = ast::parse_argument_list(arg_list_src, ParseTreeFlags::empty(), Some(&mut errors));
if !errors.is_empty() {
return get_error_text(&errors);
}
// Get the root argument list and extract arguments from it.
// Test each of these.
let arg_list: &ast::FreestandingArgumentList = ast.top().cast().unwrap();
let arg_list: &ast::FreestandingArgumentList = ast.top();
let args = &arg_list.arguments;
for arg in args.iter() {
let arg_src = arg.source(arg_list_src);

View File

@@ -1,6 +1,6 @@
// The fish parser. Contains functions for parsing and evaluating code.
use crate::ast::{self, Ast, Kind, Node};
use crate::ast::{self, Node};
use crate::builtins::shared::STATUS_ILLEGAL_CMD;
use crate::common::{
escape_string, wcs2string, CancelChecker, EscapeFlags, EscapeStringStyle, FilenameRef,
@@ -556,9 +556,7 @@ pub fn eval_parsed_source(
block_type: BlockType,
) -> EvalRes {
assert!([BlockType::top, BlockType::subst].contains(&block_type));
let Kind::JobList(job_list) = ps.ast.top().kind() else {
panic!("Expected a job list");
};
let job_list = ps.ast.top();
if !job_list.is_empty() {
// Execute the top job list.
self.eval_node(ps, job_list, io, job_group, block_type)
@@ -583,7 +581,7 @@ pub fn eval_wstr(
use crate::parse_tree::ParsedSource;
use crate::parse_util::parse_util_detect_errors_in_ast;
let mut errors = vec![];
let ast = Ast::parse(&src, ParseTreeFlags::empty(), Some(&mut errors));
let ast = ast::parse(&src, ParseTreeFlags::empty(), Some(&mut errors));
let mut errored = ast.errored();
if !errored {
errored = parse_util_detect_errors_in_ast(&ast, &src, Some(&mut errors)).is_err();
@@ -728,7 +726,7 @@ pub fn expand_argument_list(
ctx: &OperationContext<'_>,
) -> CompletionList {
// Parse the string as an argument list.
let ast = Ast::parse_argument_list(arg_list_src, ParseTreeFlags::default(), None);
let ast = ast::parse_argument_list(arg_list_src, ParseTreeFlags::default(), None);
if ast.errored() {
// Failed to parse. Here we expect to have reported any errors in test_args.
return vec![];
@@ -736,10 +734,7 @@ pub fn expand_argument_list(
// Get the root argument list and extract arguments from it.
let mut result = vec![];
let Kind::FreestandingArgumentList(list) = ast.top().kind() else {
panic!("Expected a freestanding argument list");
};
for arg in &list.arguments {
for arg in &ast.top().arguments {
let arg_src = arg.source(arg_list_src);
if matches!(
expand_string(arg_src.to_owned(), &mut result, flags, ctx, None).result,

View File

@@ -45,7 +45,7 @@
use errno::{errno, Errno};
use crate::abbrs::abbrs_match;
use crate::ast::{is_same_node, Ast, Kind};
use crate::ast::{self, is_same_node, Kind};
use crate::builtins::shared::ErrorCode;
use crate::builtins::shared::STATUS_CMD_ERROR;
use crate::builtins::shared::STATUS_CMD_OK;
@@ -5339,7 +5339,7 @@ fn extract_tokens(s: &wstr) -> Vec<PositionedToken> {
let ast_flags = ParseTreeFlags::CONTINUE_AFTER_ERROR
| ParseTreeFlags::ACCEPT_INCOMPLETE_TOKENS
| ParseTreeFlags::LEAVE_UNTERMINATED;
let ast = Ast::parse(s, ast_flags, None);
let ast = ast::parse(s, ast_flags, None);
let mut result = vec![];
let mut traversal = ast.walk();

View File

@@ -1,4 +1,4 @@
use crate::ast::{is_same_node, Ast, Node};
use crate::ast::{self, is_same_node, Node};
use crate::wchar::prelude::*;
const FISH_FUNC: &str = r#"
@@ -30,7 +30,7 @@
fn test_is_same_node() {
// is_same_node is pretty subtle! Let's check it.
let src = WString::from_str(FISH_FUNC);
let ast = Ast::parse(&src, Default::default(), None);
let ast = ast::parse(&src, Default::default(), None);
assert!(!ast.errored());
let all_nodes: Vec<&dyn Node> = ast.walk().collect();
for i in 0..all_nodes.len() {

View File

@@ -2,7 +2,7 @@
#[cfg(feature = "benchmark")]
mod bench {
extern crate test;
use crate::ast::Ast;
use crate::ast;
use crate::wchar::prelude::*;
use test::Bencher;
@@ -39,7 +39,7 @@ fn bench_ast_construction(b: &mut Bencher) {
let src = generate_fish_script();
b.bytes = (src.len() * 4) as u64; // 4 bytes per character
b.iter(|| {
let _ast = Ast::parse(&src, Default::default(), None);
let _ast = ast::parse(&src, Default::default(), None);
});
}
}

View File

@@ -31,14 +31,11 @@ macro_rules! detect_errors {
fn detect_argument_errors(src: &str) -> Result<(), ParserTestErrorBits> {
let src = WString::from_str(src);
let ast = Ast::parse_argument_list(&src, ParseTreeFlags::default(), None);
let ast = ast::parse_argument_list(&src, ParseTreeFlags::default(), None);
if ast.errored() {
return Err(ParserTestErrorBits::ERROR);
}
let Kind::FreestandingArgumentList(args) = ast.top().kind() else {
panic!("Expected free standing argument list");
};
let args = &args.arguments;
let args = &ast.top().arguments;
let first_arg = args.get(0).expect("Failed to parse an argument");
let mut errors = None;
parse_util_detect_errors_in_argument(first_arg, first_arg.source(&src), &mut errors)
@@ -326,7 +323,7 @@ fn test_new_parser_correctness() {
let _cleanup = test_init();
macro_rules! validate {
($src:expr, $ok:expr) => {
let ast = Ast::parse(L!($src), ParseTreeFlags::default(), None);
let ast = ast::parse(L!($src), ParseTreeFlags::default(), None);
assert_eq!(ast.errored(), !$ok);
};
}
@@ -403,7 +400,7 @@ fn string_for_permutation(fuzzes: &[&wstr], len: usize, permutation: usize) -> O
let mut permutation = 0;
while let Some(src) = string_for_permutation(&fuzzes, len, permutation) {
permutation += 1;
Ast::parse(&src, ParseTreeFlags::default(), None);
ast::parse(&src, ParseTreeFlags::default(), None);
}
}
}
@@ -419,7 +416,7 @@ fn test_new_parser_ll2() {
// Parse a statement, returning the command, args (joined by spaces), and the decoration. Returns
// true if successful.
fn test_1_parse_ll2(src: &wstr) -> Option<(WString, WString, StatementDecoration)> {
let ast = Ast::parse(src, ParseTreeFlags::default(), None);
let ast = ast::parse(src, ParseTreeFlags::default(), None);
if ast.errored() {
return None;
}
@@ -517,7 +514,7 @@ macro_rules! validate {
// not (issue #1240).
macro_rules! check_function_help {
($src:expr, $typ:expr) => {
let ast = Ast::parse(L!($src), ParseTreeFlags::default(), None);
let ast = ast::parse(L!($src), ParseTreeFlags::default(), None);
assert!(!ast.errored());
assert_eq!(
Traversal::new(ast.top())
@@ -541,7 +538,7 @@ fn test_new_parser_ad_hoc() {
// Ensure that 'case' terminates a job list.
let src = L!("switch foo ; case bar; case baz; end");
let ast = Ast::parse(src, ParseTreeFlags::default(), None);
let ast = ast::parse(src, ParseTreeFlags::default(), None);
assert!(!ast.errored());
// Expect two case_item_lists. The bug was that we'd
// try to run a command 'case'.
@@ -557,17 +554,17 @@ fn test_new_parser_ad_hoc() {
// leading to an infinite loop.
// By itself it should produce an error.
let ast = Ast::parse(L!("a="), ParseTreeFlags::default(), None);
let ast = ast::parse(L!("a="), ParseTreeFlags::default(), None);
assert!(ast.errored());
// If we are leaving things unterminated, this should not produce an error.
// i.e. when typing "a=" at the command line, it should be treated as valid
// because we don't want to color it as an error.
let ast = Ast::parse(L!("a="), ParseTreeFlags::LEAVE_UNTERMINATED, None);
let ast = ast::parse(L!("a="), ParseTreeFlags::LEAVE_UNTERMINATED, None);
assert!(!ast.errored());
let mut errors = vec![];
Ast::parse(
ast::parse(
L!("begin; echo ("),
ParseTreeFlags::LEAVE_UNTERMINATED,
Some(&mut errors),
@@ -576,7 +573,7 @@ fn test_new_parser_ad_hoc() {
assert!(errors[0].code == ParseErrorCode::tokenizer_unterminated_subshell);
errors.clear();
Ast::parse(
ast::parse(
L!("for x in ("),
ParseTreeFlags::LEAVE_UNTERMINATED,
Some(&mut errors),
@@ -585,7 +582,7 @@ fn test_new_parser_ad_hoc() {
assert!(errors[0].code == ParseErrorCode::tokenizer_unterminated_subshell);
errors.clear();
Ast::parse(
ast::parse(
L!("begin; echo '"),
ParseTreeFlags::LEAVE_UNTERMINATED,
Some(&mut errors),
@@ -601,7 +598,7 @@ fn test_new_parser_errors() {
macro_rules! validate {
($src:expr, $expected_code:expr) => {
let mut errors = vec![];
let ast = Ast::parse(L!($src), ParseTreeFlags::default(), Some(&mut errors));
let ast = ast::parse(L!($src), ParseTreeFlags::default(), Some(&mut errors));
assert!(ast.errored());
assert_eq!(
errors.into_iter().map(|e| e.code).collect::<Vec<_>>(),
@@ -835,7 +832,7 @@ struct TrueSemiAstTester<'a> {
impl<'a> TrueSemiAstTester<'a> {
const TRUE_SEMI: &'static wstr = L!("true;");
fn new(ast: &'a Ast) -> Self {
let job_list: &JobList = ast.top().cast().unwrap();
let job_list: &JobList = ast.top();
let job_conjunction = &job_list[0];
let job_pipeline = &job_conjunction.job;
let variable_assignment_list = &job_pipeline.variables;
@@ -895,9 +892,8 @@ fn get_parents<'s>(&'s self, node: &'a dyn Node) -> impl Iterator<Item = &'a dyn
#[test]
fn test_ast() {
// Light testing of the AST and traversals.
let ast = Ast::parse(TrueSemiAstTester::TRUE_SEMI, ParseTreeFlags::empty(), None);
let ast = ast::parse(TrueSemiAstTester::TRUE_SEMI, ParseTreeFlags::empty(), None);
let tester = TrueSemiAstTester::new(&ast);
assert!(ast.top().cast::<JobList>().is_some(), "Expected job_list");
// Walk the AST and collect all nodes.
// See is_same_node comments for why we can't use assert_eq! here.
@@ -957,7 +953,7 @@ fn test_ast() {
#[should_panic]
fn test_traversal_skip_children_panics() {
// Test that we panic if we try to skip children of a node that is not the current node.
let ast = Ast::parse(L!("true;"), ParseTreeFlags::empty(), None);
let ast = ast::parse(L!("true;"), ParseTreeFlags::empty(), None);
let mut traversal = ast.walk();
while let Some(node) = traversal.next() {
if node.typ() == ast::Type::decorated_statement {
@@ -971,7 +967,7 @@ fn test_traversal_skip_children_panics() {
#[should_panic]
fn test_traversal_parent_panics() {
// Can only get the parent of nodes still on the stack.
let ast = Ast::parse(L!("true;"), ParseTreeFlags::empty(), None);
let ast = ast::parse(L!("true;"), ParseTreeFlags::empty(), None);
let mut traversal = ast.walk();
let mut decorated_statement = None;
while let Some(node) = traversal.next() {