h/fish-shell
1
Fork 0
mirror of https://github.com/fish-shell/fish-shell.git synced 2026-05-23 13:11:15 -03:00
Code Issues Packages Projects Releases Wiki Activity

Rework fish AST implementation

Browse Source 
This merges a large set of changes to the fish AST, with the intention of
making the code simpler.

There's no expected user-visible changes here, except for some minor
changes in the output of `fish_indent --dump-parse-tree`.

Ast parsing is about 50% faster measured via
`cargo +nightly bench  --features=benchmark bench_ast_construction`
and also uses less memory due to some size optimization.

The biggest change is removing the `Type` notion from `Node`. Previously
each Node had an integer type identified with it, like Type::Argument. This
was a relic from C++: types were natural in C++ and we could use LLVM-style
RTTI to identify Nodes, leveraging the fact that C++ has inheritance and so
Type could be at the same location in each Node.

This proved quite awkward in Rust which does not have inheritance. So
instead we switch to a new notion, Kind:

    pub enum Kind<'a> {
        Redirection(&'a Redirection),
        Token(&'a dyn Token),
        Keyword(&'a dyn Keyword),
        VariableAssignment(&'a VariableAssignment),
                ...

and a `&dyn Node` can now return its Kind. Basically leveraging Rust's enum
types.

Interesting lesson about the optimal way to construct ASTs in both
languages.
This commit is contained in:
Peter Ammon
2025-05-04 19:45:36 -07:00
parent 111922b60f fe10f65587
commit c993fd022c
15 changed files with 1107 additions and 2281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2637
src/ast.rs
View File

File diff suppressed because it is too large Load Diff

4
src/bin/fish.rs
View File

@@ -22,7 +22,7 @@
#![allow(clippy::uninlined_format_args)]
use fish::{
ast::Ast,
ast,
builtins::{
fish_indent, fish_key_reader,
shared::{
@@ -226,7 +226,7 @@ fn run_command_list(parser: &Parser, cmds: &[OsString]) -> Result<(), libc::c_in
let cmd_wcs = str2wcstring(cmd.as_bytes());
let mut errors = ParseErrorList::new();
let ast = Ast::parse(&cmd_wcs, ParseTreeFlags::empty(), Some(&mut errors));
let ast = ast::parse(&cmd_wcs, ParseTreeFlags::empty(), Some(&mut errors));
let errored = ast.errored() || {
parse_util_detect_errors_in_ast(&ast, &cmd_wcs, Some(&mut errors)).is_err()
};

6
src/builtins/commandline.rs
View File

@@ -1,5 +1,5 @@
use super::prelude::*;
use crate::ast::{Ast, Leaf};
use crate::ast::{self, Kind, Leaf};
use crate::common::{unescape_string, UnescapeFlags, UnescapeStringStyle};
use crate::complete::Completion;
use crate::expand::{expand_string, ExpandFlags, ExpandResultCode};
@@ -108,7 +108,7 @@ fn strip_dollar_prefixes(insert_mode: AppendMode, prefix: &wstr, insert: &wstr)
}
insert.find(L!("$ "))?; // Early return.
let source = prefix.to_owned() + insert;
let ast = Ast::parse(
let ast = ast::parse(
&source,
ParseTreeFlags::ACCEPT_INCOMPLETE_TOKENS | ParseTreeFlags::LEAVE_UNTERMINATED,
None,
@@ -116,7 +116,7 @@ fn strip_dollar_prefixes(insert_mode: AppendMode, prefix: &wstr, insert: &wstr)
let mut stripped = WString::new();
let mut have = prefix.len();
for node in ast.walk() {
let Some(ds) = node.as_decorated_statement() else {
let Kind::DecoratedStatement(ds) = node.kind() else {
continue;
};
let Some(range) = ds.command.range() else {

95
src/builtins/fish_indent.rs
View File

@@ -15,9 +15,7 @@
use libc::LC_ALL;
use super::prelude::*;
use crate::ast::{
self, Ast, Category, Leaf, List, Node, NodeVisitor, SourceRangeList, Traversal, Type,
};
use crate::ast::{self, Ast, Kind, Leaf, Node, NodeVisitor, SourceRangeList, Traversal};
use crate::common::{
str2wcstring, unescape_string, wcs2string, UnescapeFlags, UnescapeStringStyle, PROGRAM_NAME,
};
@@ -96,7 +94,6 @@ struct AstSizeMetrics {
/// Note tokens and keywords are also counted as leaves.
branch_count: usize,
leaf_count: usize,
list_count: usize,
token_count: usize,
keyword_count: usize,
// An estimate of the total allocated size of the ast in bytes.
@@ -109,7 +106,6 @@ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
writeln!(f, " nodes: {}", self.node_count)?;
writeln!(f, " branches: {}", self.branch_count)?;
writeln!(f, " leaves: {}", self.leaf_count)?;
writeln!(f, " lists: {}", self.list_count)?;
writeln!(f, " tokens: {}", self.token_count)?;
writeln!(f, " keywords: {}", self.keyword_count)?;
@@ -127,10 +123,10 @@ impl<'a> NodeVisitor<'a> for AstSizeMetrics {
fn visit(&mut self, node: &'a dyn Node) {
self.node_count += 1;
self.memory_size += node.self_memory_size();
match node.category() {
Category::branch => self.branch_count += 1,
Category::leaf => self.leaf_count += 1,
Category::list => self.list_count += 1,
if node.as_leaf().is_some() {
self.leaf_count += 1;
} else {
self.branch_count += 1; // treating lists as branches
}
if node.as_token().is_some() {
self.token_count += 1;
@@ -138,7 +134,7 @@ fn visit(&mut self, node: &'a dyn Node) {
if node.as_keyword().is_some() {
self.keyword_count += 1;
}
node.accept(self, false);
node.accept(self);
}
}
@@ -222,7 +218,7 @@ fn compute_gaps(&self) -> Vec<SourceRange> {
// Collect the token ranges into a list.
let mut tok_ranges = vec![];
for node in Traversal::new(self.ast.top()) {
if node.category() == Category::leaf {
if let Some(node) = node.as_leaf() {
let r = node.source_range();
if r.length() > 0 {
tok_ranges.push(r);
@@ -268,10 +264,10 @@ fn compute_preferred_semi_locations(&self) -> Vec<usize> {
// See if we have a condition and an andor_job_list.
let condition;
let andors;
if let Some(ifc) = node.as_if_clause() {
if let Kind::IfClause(ifc) = node.kind() {
condition = ifc.condition.semi_nl.as_ref();
andors = &ifc.andor_tail;
} else if let Some(wc) = node.as_while_header() {
} else if let Kind::WhileHeader(wc) = node.kind() {
condition = wc.condition.semi_nl.as_ref();
andors = &wc.andor_tail;
} else {
@@ -279,10 +275,10 @@ fn compute_preferred_semi_locations(&self) -> Vec<usize> {
}
// If there is no and-or tail then we always use a newline.
if andors.count() > 0 {
if !andors.is_empty() {
condition.map(&mut mark_semi_from_input);
// Mark all but last of the andor list.
for andor in andors.iter().take(andors.count() - 1) {
for andor in andors.iter().take(andors.len() - 1) {
mark_semi_from_input(andor.job.semi_nl.as_ref().unwrap());
}
}
@@ -290,7 +286,7 @@ fn compute_preferred_semi_locations(&self) -> Vec<usize> {
// `{ x; y; }` gets semis if the input uses semis and it spans only one line.
for node in Traversal::new(self.ast.top()) {
let Some(brace_statement) = node.as_brace_statement() else {
let Kind::BraceStatement(brace_statement) = node.kind() else {
continue;
};
if self
@@ -307,7 +303,7 @@ fn compute_preferred_semi_locations(&self) -> Vec<usize> {
// `x ; and y` gets semis if it has them already, and they are on the same line.
for node in Traversal::new(self.ast.top()) {
let Some(job_list) = node.as_job_list() else {
let Kind::JobList(job_list) = node.kind() else {
continue;
};
let mut prev_job_semi_nl = None;
@@ -355,7 +351,7 @@ fn compute_multi_line_brace_statement_locations(&self) -> Vec<usize> {
.collect();
let mut next_newline = 0;
for node in Traversal::new(self.ast.top()) {
let Some(brace_statement) = node.as_brace_statement() else {
let Kind::BraceStatement(brace_statement) = node.kind() else {
continue;
};
while next_newline != newline_offsets.len()
@@ -385,14 +381,14 @@ fn indent(&self, index: usize) -> usize {
// Return gap text flags for the gap text that comes *before* a given node type.
fn gap_text_flags_before_node(&self, node: &dyn Node) -> GapFlags {
let mut result = GapFlags::default();
match node.typ() {
match node.kind() {
// Allow escaped newlines before leaf nodes that can be part of a long command.
Type::argument | Type::redirection | Type::variable_assignment => {
Kind::Argument(_) | Kind::Redirection(_) | Kind::VariableAssignment(_) => {
result.allow_escaped_newlines = true
}
Type::token_base => {
Kind::Token(token) => {
// Allow escaped newlines before && and ||, and also pipes.
match node.as_token().unwrap().token_type() {
match token.token_type() {
ParseTokenType::andand | ParseTokenType::oror | ParseTokenType::pipe => {
result.allow_escaped_newlines = true;
}
@@ -400,21 +396,21 @@ fn gap_text_flags_before_node(&self, node: &dyn Node) -> GapFlags {
// Allow escaped newlines before commands that follow a variable assignment
// since both can be long (#7955).
let p = self.traversal.parent(node);
if p.typ() != Type::decorated_statement {
if !matches!(p.kind(), Kind::DecoratedStatement(_)) {
return result;
}
let p = self.traversal.parent(p);
assert_eq!(p.typ(), Type::statement);
assert!(matches!(p.kind(), Kind::Statement(_)));
let p = self.traversal.parent(p);
if let Some(job) = p.as_job_pipeline() {
if let Kind::JobPipeline(job) = p.kind() {
if !job.variables.is_empty() {
result.allow_escaped_newlines = true;
}
} else if let Some(job_cnt) = p.as_job_continuation() {
} else if let Kind::JobContinuation(job_cnt) = p.kind() {
if !job_cnt.variables.is_empty() {
result.allow_escaped_newlines = true;
}
} else if let Some(not_stmt) = p.as_not_statement() {
} else if let Kind::NotStatement(not_stmt) = p.kind() {
if !not_stmt.variables.is_empty() {
result.allow_escaped_newlines = true;
}
@@ -731,14 +727,12 @@ fn visit_semi_nl(&mut self, node: &dyn ast::Token) {
}
fn is_multi_line_brace(&self, node: &dyn ast::Token) -> bool {
self.traversal
.parent(node.as_node())
.as_brace_statement()
.is_some_and(|brace_statement| {
self.multi_line_brace_statement_locations
.binary_search(&brace_statement.source_range().start())
.is_ok()
})
let Kind::BraceStatement(brace) = self.traversal.parent(node.as_node()).kind() else {
return false;
};
self.multi_line_brace_statement_locations
.binary_search(&brace.source_range().start())
.is_ok()
}
fn visit_left_brace(&mut self, node: &dyn ast::Token) {
let range = node.source_range();
@@ -837,29 +831,30 @@ fn prettify_traversal(&mut self) {
}
continue;
}
match node.typ() {
Type::argument | Type::variable_assignment => {
match node.kind() {
Kind::Argument(_) | Kind::VariableAssignment(_) => {
self.emit_node_text(node);
self.traversal.skip_children(node);
}
Type::redirection => {
self.visit_redirection(node.as_redirection().unwrap());
Kind::Redirection(node) => {
self.visit_redirection(node);
self.traversal.skip_children(node);
}
Type::maybe_newlines => {
self.visit_maybe_newlines(node.as_maybe_newlines().unwrap());
Kind::MaybeNewlines(node) => {
self.visit_maybe_newlines(node);
self.traversal.skip_children(node);
}
Type::begin_header => {
self.visit_begin_header(node.as_begin_header().unwrap());
Kind::BeginHeader(node) => {
self.visit_begin_header(node);
self.traversal.skip_children(node);
}
_ => {
// For branch and list nodes, default is to visit their children.
if [Category::branch, Category::list].contains(&node.category()) {
continue;
}
panic!("unexpected node type");
// Default is to visit children. We expect all leaves to have been handled above.
assert!(
node.as_leaf().is_none(),
"Should have handled all leaf nodes"
);
}
}
}
@@ -1246,7 +1241,7 @@ struct TokenRange {
// Entry point for prettification.
fn prettify(streams: &mut IoStreams, src: &wstr, do_indent: bool) -> WString {
if DUMP_PARSE_TREE.load() {
let ast = Ast::parse(
let ast = ast::parse(
src,
ParseTreeFlags::LEAVE_UNTERMINATED
| ParseTreeFlags::INCLUDE_COMMENTS
@@ -1261,7 +1256,7 @@ fn prettify(streams: &mut IoStreams, src: &wstr, do_indent: bool) -> WString {
metrics.visit(ast.top());
streams.err.appendln(format!("{}", metrics));
}
let ast = Ast::parse(src, parse_flags(), None);
let ast = ast::parse(src, parse_flags(), None);
let mut printer = PrettyPrinter::new(src, &ast, do_indent);
printer.prettify()
}

45
src/highlight/highlight.rs
View File

@@ -1,9 +1,8 @@
//! Functions for syntax highlighting.
use crate::abbrs::{self, with_abbrs};
use crate::ast::{
self, Argument, Ast, BlockStatement, BlockStatementHeaderVariant, BraceStatement,
DecoratedStatement, Keyword, List, Node, NodeVisitor, Redirection, Token, Type,
VariableAssignment,
self, Argument, BlockStatement, BlockStatementHeader, BraceStatement, DecoratedStatement,
Keyword, Kind, Node, NodeVisitor, Redirection, Token, VariableAssignment,
};
use crate::builtins::shared::builtin_exists;
use crate::color::Color;
@@ -272,15 +271,16 @@ fn autosuggest_parse_command(
buff: &wstr,
ctx: &OperationContext<'_>,
) -> Option<(WString, WString)> {
let ast = Ast::parse(
let ast = ast::parse(
buff,
ParseTreeFlags::CONTINUE_AFTER_ERROR | ParseTreeFlags::ACCEPT_INCOMPLETE_TOKENS,
None,
);
// Find the first statement.
let jc = ast.top().as_job_list().unwrap().get(0)?;
let first_statement = jc.job.statement.contents.as_decorated_statement()?;
let job_list: &ast::JobList = ast.top();
let jc = job_list.get(0)?;
let first_statement = jc.job.statement.as_decorated_statement()?;
if let Some(expanded_command) = statement_get_expanded_command(buff, first_statement, ctx) {
let mut arg = WString::new();
@@ -709,7 +709,7 @@ pub fn highlight(&mut self) -> ColorArray {
| ParseTreeFlags::ACCEPT_INCOMPLETE_TOKENS
| ParseTreeFlags::LEAVE_UNTERMINATED
| ParseTreeFlags::SHOW_EXTRA_SEMIS;
let ast = Ast::parse(self.buff, ast_flags, None);
let ast = ast::parse(self.buff, ast_flags, None);
self.visit_children(ast.top());
if self.ctx.check_cancel() {
@@ -832,7 +832,7 @@ fn color_range(&mut self, range: SourceRange, color: HighlightSpec) {
// Visit the children of a node.
fn visit_children(&mut self, node: &dyn Node) {
node.accept(self, false);
node.accept(self);
}
// AST visitor implementations.
fn visit_keyword(&mut self, node: &dyn Keyword) {
@@ -1044,15 +1044,14 @@ fn visit_decorated_statement(&mut self, stmt: &DecoratedStatement) {
}
fn visit_block_statement(&mut self, block: &BlockStatement) {
match &block.header {
BlockStatementHeaderVariant::None => panic!(),
BlockStatementHeaderVariant::ForHeader(node) => self.visit(node),
BlockStatementHeaderVariant::WhileHeader(node) => self.visit(node),
BlockStatementHeaderVariant::FunctionHeader(node) => self.visit(node),
BlockStatementHeaderVariant::BeginHeader(node) => self.visit(node),
BlockStatementHeader::For(node) => self.visit(node),
BlockStatementHeader::While(node) => self.visit(node),
BlockStatementHeader::Function(node) => self.visit(node),
BlockStatementHeader::Begin(node) => self.visit(node),
}
self.visit(&block.args_or_redirs);
let pending_variables_count = self.pending_variables.len();
if let Some(fh) = block.header.as_for_header() {
if let BlockStatementHeader::For(fh) = &block.header {
let var_name = fh.var_name.source(self.buff);
self.pending_variables.push(var_name);
}
@@ -1114,17 +1113,13 @@ fn visit(&mut self, node: &'a dyn Node) {
self.visit_token(token);
return;
}
match node.typ() {
Type::argument => self.visit_argument(node.as_argument().unwrap(), false, true),
Type::redirection => self.visit_redirection(node.as_redirection().unwrap()),
Type::variable_assignment => {
self.visit_variable_assignment(node.as_variable_assignment().unwrap())
}
Type::decorated_statement => {
self.visit_decorated_statement(node.as_decorated_statement().unwrap())
}
Type::block_statement => self.visit_block_statement(node.as_block_statement().unwrap()),
Type::brace_statement => self.visit_brace_statement(node.as_brace_statement().unwrap()),
match node.kind() {
Kind::Argument(node) => self.visit_argument(node, false, true),
Kind::Redirection(node) => self.visit_redirection(node),
Kind::VariableAssignment(node) => self.visit_variable_assignment(node),
Kind::DecoratedStatement(node) => self.visit_decorated_statement(node),
Kind::BlockStatement(node) => self.visit_block_statement(node),
Kind::BraceStatement(node) => self.visit_brace_statement(node),
// Default implementation is to just visit children.
_ => self.visit_children(node),
}

10
src/history.rs
View File

@@ -47,7 +47,7 @@
use rand::Rng;
use crate::{
ast::{Ast, Node},
ast::{self, Kind, Node},
common::{
str2wcstring, unescape_string, valid_var_name, wcs2zstring, CancelChecker,
UnescapeStringStyle,
@@ -1496,7 +1496,7 @@ fn should_import_bash_history_line(line: &wstr) -> bool {
}
}
if Ast::parse(line, ParseTreeFlags::empty(), None).errored() {
if ast::parse(line, ParseTreeFlags::empty(), None).errored() {
return false;
}
@@ -1581,16 +1581,16 @@ pub fn add_pending_with_file_detection(
// Find all arguments that look like they could be file paths.
let mut needs_sync_write = false;
let ast = Ast::parse(s, ParseTreeFlags::empty(), None);
let ast = ast::parse(s, ParseTreeFlags::empty(), None);
let mut potential_paths = Vec::new();
for node in ast.walk() {
if let Some(arg) = node.as_argument() {
if let Kind::Argument(arg) = node.kind() {
let potential_path = arg.source(s);
if string_could_be_path(potential_path) {
potential_paths.push(potential_path.to_owned());
}
} else if let Some(stmt) = node.as_decorated_statement() {
} else if let Kind::DecoratedStatement(stmt) = node.kind() {
// Hack hack hack - if the command is likely to trigger an exit, then don't do
// background file detection, because we won't be able to write it to our history file
// before we exit.

25
src/parse_constants.rs
View File

@@ -38,24 +38,15 @@ pub struct ParserTestErrorBits: u8 {
}
/// A range of source code.
#[derive(PartialEq, Eq, Clone, Copy, Debug)]
#[derive(PartialEq, Eq, Clone, Copy, Debug, Default)]
pub struct SourceRange {
pub start: u32,
pub length: u32,
}
impl Default for SourceRange {
fn default() -> Self {
SourceRange {
start: 0,
length: 0,
}
}
}
impl SourceRange {
pub fn as_usize(&self) -> std::ops::Range<usize> {
(*self).into()
pub fn as_usize(self) -> std::ops::Range<usize> {
self.into()
}
}
@@ -156,20 +147,20 @@ pub fn new(start: usize, length: usize) -> Self {
length: length.try_into().unwrap(),
}
}
pub fn start(&self) -> usize {
pub fn start(self) -> usize {
self.start.try_into().unwrap()
}
pub fn length(&self) -> usize {
pub fn length(self) -> usize {
self.length.try_into().unwrap()
}
pub fn end(&self) -> usize {
pub fn end(self) -> usize {
self.start
.checked_add(self.length)
.expect("Overflow")
.try_into()
.unwrap()
}
pub fn combine(&self, other: Self) -> Self {
pub fn combine(self, other: Self) -> Self {
let start = std::cmp::min(self.start, other.start);
SourceRange {
start,
@@ -182,7 +173,7 @@ pub fn combine(&self, other: Self) -> Self {
}
// Return true if a location is in this range, including one-past-the-end.
pub fn contains_inclusive(&self, loc: usize) -> bool {
pub fn contains_inclusive(self, loc: usize) -> bool {
self.start() <= loc && loc - self.start() <= self.length()
}
}

193
src/parse_execution.rs
View File

@@ -1,8 +1,7 @@
//! Provides the "linkage" between an ast and actual execution structures (job_t, etc.).
use crate::ast::{
self, unescape_keyword, BlockStatementHeaderVariant, Keyword, Leaf, List, Node,
StatementVariant, Token,
self, unescape_keyword, BlockStatementHeader, Keyword, Leaf, Node, Statement, Token,
};
use crate::builtins;
use crate::builtins::shared::{
@@ -140,13 +139,9 @@ pub fn eval_node(
node: &'a dyn Node,
associated_block: Option<BlockId>,
) -> EndExecutionReason {
match node.typ() {
ast::Type::statement => {
self.eval_statement(ctx, node.as_statement().unwrap(), associated_block)
}
ast::Type::job_list => {
self.eval_job_list(ctx, node.as_job_list().unwrap(), associated_block.unwrap())
}
match node.kind() {
ast::Kind::Statement(node) => self.eval_statement(ctx, node, associated_block),
ast::Kind::JobList(node) => self.eval_job_list(ctx, node, associated_block.unwrap()),
_ => unreachable!(),
}
}
@@ -160,22 +155,14 @@ fn eval_statement(
associated_block: Option<BlockId>,
) -> EndExecutionReason {
// Note we only expect block-style statements here. No not statements.
match &statement.contents {
StatementVariant::BlockStatement(block) => {
self.run_block_statement(ctx, block, associated_block)
}
StatementVariant::BraceStatement(brace_statement) => {
match &statement {
Statement::Block(block) => self.run_block_statement(ctx, block, associated_block),
Statement::Brace(brace_statement) => {
self.run_begin_statement(ctx, &brace_statement.jobs)
}
StatementVariant::IfStatement(ifstat) => {
self.run_if_statement(ctx, ifstat, associated_block)
}
StatementVariant::SwitchStatement(switchstat) => {
self.run_switch_statement(ctx, switchstat)
}
StatementVariant::DecoratedStatement(_)
| StatementVariant::NotStatement(_)
| StatementVariant::None => panic!(),
Statement::If(ifstat) => self.run_if_statement(ctx, ifstat, associated_block),
Statement::Switch(switchstat) => self.run_switch_statement(ctx, switchstat),
Statement::Decorated(_) | Statement::Not(_) => panic!(),
}
}
@@ -418,7 +405,7 @@ fn infinite_recursive_statement_in_job_list<'b>(
// Helper to return if a statement is infinitely recursive in this function.
let statement_recurses = |stat: &'b ast::Statement| -> Option<&'b ast::DecoratedStatement> {
// Ignore non-decorated statements like `if`, etc.
let StatementVariant::DecoratedStatement(dc) = &stat.contents else {
let Statement::Decorated(dc) = &stat else {
return None;
};
@@ -560,18 +547,16 @@ fn job_is_simple_block(&self, job: &ast::JobPipeline) -> bool {
let no_redirs =
|list: &ast::ArgumentOrRedirectionList| !list.iter().any(|val| val.is_redirection());
// Check if we're a block statement with redirections. We do it this obnoxious way to preserve
// type safety (in case we add more specific statement types).
match &job.statement.contents {
StatementVariant::BlockStatement(stmt) => no_redirs(&stmt.args_or_redirs),
StatementVariant::BraceStatement(stmt) => no_redirs(&stmt.args_or_redirs),
StatementVariant::SwitchStatement(stmt) => no_redirs(&stmt.args_or_redirs),
StatementVariant::IfStatement(stmt) => no_redirs(&stmt.args_or_redirs),
StatementVariant::NotStatement(_) | StatementVariant::DecoratedStatement(_) => {
// Check if we're a block statement with redirections.
match &job.statement {
Statement::Block(stmt) => no_redirs(&stmt.args_or_redirs),
Statement::Brace(stmt) => no_redirs(&stmt.args_or_redirs),
Statement::Switch(stmt) => no_redirs(&stmt.args_or_redirs),
Statement::If(stmt) => no_redirs(&stmt.args_or_redirs),
Statement::Not(_) | Statement::Decorated(_) => {
// not block statements
false
}
StatementVariant::None => panic!(),
}
}
@@ -664,9 +649,6 @@ fn populate_job_process(
statement: &ast::Statement,
variable_assignments: &ast::VariableAssignmentList,
) -> EndExecutionReason {
// Get the "specific statement" which is boolean / block / if / switch / decorated.
let specific_statement = &statement.contents;
let mut block = None;
let result =
self.apply_variable_assignments(ctx, Some(proc), variable_assignments, &mut block);
@@ -679,20 +661,16 @@ fn populate_job_process(
return result;
}
match &specific_statement {
StatementVariant::NotStatement(not_statement) => {
match &statement {
Statement::Not(not_statement) => {
self.populate_not_process(ctx, job, proc, not_statement)
}
StatementVariant::BlockStatement(_)
| StatementVariant::BraceStatement(_)
| StatementVariant::IfStatement(_)
| StatementVariant::SwitchStatement(_) => {
self.populate_block_process(ctx, proc, statement, specific_statement)
Statement::Block(_) | Statement::Brace(_) | Statement::If(_) | Statement::Switch(_) => {
self.populate_block_process(ctx, proc, statement)
}
StatementVariant::DecoratedStatement(decorated_statement) => {
Statement::Decorated(decorated_statement) => {
self.populate_plain_process(ctx, proc, decorated_statement)
}
StatementVariant::None => panic!(),
}
}
@@ -841,17 +819,16 @@ fn populate_block_process(
ctx: &OperationContext<'_>,
proc: &mut Process,
statement: &ast::Statement,
specific_statement: &ast::StatementVariant,
) -> EndExecutionReason {
// We handle block statements by creating process_type_t::block_node, that will bounce back to
// We handle block statements by creating ProcessType::block_node, that will bounce back to
// us when it's time to execute them.
// Get the argument or redirections list.
// TODO: args_or_redirs should be available without resolving the statement type.
let args_or_redirs = match specific_statement {
StatementVariant::BlockStatement(block_statement) => &block_statement.args_or_redirs,
StatementVariant::BraceStatement(brace_statement) => &brace_statement.args_or_redirs,
StatementVariant::IfStatement(if_statement) => &if_statement.args_or_redirs,
StatementVariant::SwitchStatement(switch_statement) => &switch_statement.args_or_redirs,
let args_or_redirs = match statement {
Statement::Block(block_statement) => &block_statement.args_or_redirs,
Statement::Brace(brace_statement) => &brace_statement.args_or_redirs,
Statement::If(if_statement) => &if_statement.args_or_redirs,
Statement::Switch(switch_statement) => &switch_statement.args_or_redirs,
_ => panic!("Unexpected block node type"),
};
@@ -876,17 +853,12 @@ fn run_block_statement(
let bh = &statement.header;
let contents = &statement.jobs;
match bh {
BlockStatementHeaderVariant::ForHeader(fh) => self.run_for_statement(ctx, fh, contents),
BlockStatementHeaderVariant::WhileHeader(wh) => {
BlockStatementHeader::For(fh) => self.run_for_statement(ctx, fh, contents),
BlockStatementHeader::While(wh) => {
self.run_while_statement(ctx, wh, contents, associated_block)
}
BlockStatementHeaderVariant::FunctionHeader(fh) => {
self.run_function_statement(ctx, statement, fh)
}
BlockStatementHeaderVariant::BeginHeader(_bh) => {
self.run_begin_statement(ctx, contents)
}
BlockStatementHeaderVariant::None => panic!(),
BlockStatementHeader::Function(fh) => self.run_function_statement(ctx, statement, fh),
BlockStatementHeader::Begin(_bh) => self.run_begin_statement(ctx, contents),
}
}
@@ -1594,29 +1566,23 @@ fn run_1_job(
}
});
let specific_statement = &job_node.statement.contents;
assert!(specific_statement_type_is_redirectable_block(
specific_statement
));
let statement = &job_node.statement;
assert!(statement_is_redirectable_block(statement));
if result == EndExecutionReason::ok {
result = match &specific_statement {
StatementVariant::BlockStatement(block_statement) => {
result = match statement {
Statement::Block(block_statement) => {
self.run_block_statement(ctx, block_statement, associated_block)
}
StatementVariant::BraceStatement(brace_statement) => {
Statement::Brace(brace_statement) => {
self.run_begin_statement(ctx, &brace_statement.jobs)
}
StatementVariant::IfStatement(ifstmt) => {
self.run_if_statement(ctx, ifstmt, associated_block)
}
StatementVariant::SwitchStatement(switchstmt) => {
self.run_switch_statement(ctx, switchstmt)
}
Statement::If(ifstmt) => self.run_if_statement(ctx, ifstmt, associated_block),
Statement::Switch(switchstmt) => self.run_switch_statement(ctx, switchstmt),
// Other types should be impossible due to the
// specific_statement_type_is_redirectable_block check.
StatementVariant::NotStatement(_)
| StatementVariant::DecoratedStatement(_)
| StatementVariant::None => panic!(),
// statement_is_redirectable_block check.
Statement::Not(_) | Statement::Decorated(_) => {
panic!()
}
};
}
@@ -1627,7 +1593,7 @@ fn run_1_job(
profile_item.duration = ProfileItem::now() - start_time;
profile_item.level = ctx.parser().scope().eval_level;
profile_item.cmd =
profiling_cmd_name_for_redirectable_block(specific_statement, self.pstree());
profiling_cmd_name_for_redirectable_block(statement, self.pstree());
profile_item.skipped = false;
}
@@ -1931,56 +1897,35 @@ enum Globspec {
}
type AstArgsList<'a> = Vec<&'a ast::Argument>;
/// These are the specific statement types that support redirections.
fn type_is_redirectable_block(typ: ast::Type) -> bool {
[
ast::Type::block_statement,
ast::Type::brace_statement,
ast::Type::if_statement,
ast::Type::switch_statement,
]
.contains(&typ)
}
fn specific_statement_type_is_redirectable_block(node: &ast::StatementVariant) -> bool {
type_is_redirectable_block(node.typ())
fn statement_is_redirectable_block(node: &ast::Statement) -> bool {
match node {
Statement::Decorated(_) | Statement::Not(_) => false,
Statement::Block(_) | Statement::Brace(_) | Statement::If(_) | Statement::Switch(_) => true,
}
}
/// Get the name of a redirectable block, for profiling purposes.
fn profiling_cmd_name_for_redirectable_block(
node: &ast::StatementVariant,
node: &ast::Statement,
pstree: &ParsedSourceRef,
) -> WString {
assert!(specific_statement_type_is_redirectable_block(node));
assert!(statement_is_redirectable_block(node));
let source_range = node.try_source_range().expect("No source range for block");
let src_end = match node {
StatementVariant::BlockStatement(block_statement) => {
Statement::Block(block_statement) => {
let block_header = &block_statement.header;
match block_header {
BlockStatementHeaderVariant::ForHeader(for_header) => {
for_header.semi_nl.source_range().start()
}
BlockStatementHeaderVariant::WhileHeader(while_header) => {
while_header.condition.source_range().start()
}
BlockStatementHeaderVariant::FunctionHeader(function_header) => {
function_header.semi_nl.source_range().start()
}
BlockStatementHeaderVariant::BeginHeader(begin_header) => {
begin_header.kw_begin.source_range().start()
}
BlockStatementHeaderVariant::None => panic!("Unexpected block header type"),
BlockStatementHeader::For(node) => node.semi_nl.source_range().start(),
BlockStatementHeader::While(node) => node.condition.source_range().start(),
BlockStatementHeader::Function(node) => node.semi_nl.source_range().start(),
BlockStatementHeader::Begin(node) => node.kw_begin.source_range().start(),
}
}
StatementVariant::BraceStatement(brace_statement) => {
brace_statement.left_brace.source_range().start()
}
StatementVariant::IfStatement(ifstmt) => {
ifstmt.if_clause.condition.job.source_range().end()
}
StatementVariant::SwitchStatement(switchstmt) => switchstmt.semi_nl.source_range().start(),
Statement::Brace(brace_statement) => brace_statement.left_brace.source_range().start(),
Statement::If(ifstmt) => ifstmt.if_clause.condition.job.source_range().end(),
Statement::Switch(switchstmt) => switchstmt.semi_nl.source_range().start(),
_ => {
panic!("Not a redirectable block_type");
}
@@ -2011,17 +1956,19 @@ fn job_node_wants_timing(job_node: &ast::JobPipeline) -> bool {
}
// Helper to return true if a node is 'not time ...' or 'not not time...' or...
let is_timed_not_statement = |mut stat: &ast::Statement| loop {
match &stat.contents {
StatementVariant::NotStatement(ns) => {
if ns.time.is_some() {
return true;
fn is_timed_not_statement(mut stat: &ast::Statement) -> bool {
loop {
match &stat {
Statement::Not(ns) => {
if ns.time.is_some() {
return true;
}
stat = &ns.contents;
}
stat = &ns.contents;
_ => return false,
}
_ => return false,
}
};
}
// Do we have a 'not time ...' anywhere in our pipeline?
if is_timed_not_statement(&job_node.statement) {

4
src/parse_tree.rs
View File

@@ -4,7 +4,7 @@
use std::pin::Pin;
use std::sync::Arc;
use crate::ast::{Ast, Node};
use crate::ast::{self, Ast, Node};
use crate::common::{assert_send, assert_sync};
use crate::parse_constants::{
token_type_user_presentable_description, ParseErrorCode, ParseErrorList, ParseKeyword,
@@ -187,7 +187,7 @@ pub fn parse_source(
flags: ParseTreeFlags,
errors: Option<&mut ParseErrorList>,
) -> Option<ParsedSourceRef> {
let ast = Ast::parse(&src, flags, errors);
let ast = ast::parse(&src, flags, errors);
if ast.errored() && !flags.contains(ParseTreeFlags::CONTINUE_AFTER_ERROR) {
None
} else {

270
src/parse_util.rs
View File

@@ -1,6 +1,6 @@
//! Various mostly unrelated utility functions related to parsing, loading and evaluating fish code.
use crate::ast::{
self, is_same_node, Ast, Keyword, Leaf, List, Node, NodeVisitor, Token, Traversal,
self, is_same_node, Ast, Keyword, Kind, Leaf, Node, NodeVisitor, Token, Traversal,
};
use crate::builtins::shared::builtin_exists;
use crate::common::{
@@ -767,7 +767,7 @@ fn compute_indents(src: &wstr, initial_indent: i32) -> Vec<i32> {
// the last node we visited becomes the input indent of the next. I.e. in the case of 'switch
// foo ; cas', we get an invalid parse tree (since 'cas' is not valid) but we indent it as if it
// were a case item list.
let ast = Ast::parse(
let ast = ast::parse(
src,
ParseTreeFlags::CONTINUE_AFTER_ERROR
| ParseTreeFlags::INCLUDE_COMMENTS
@@ -995,22 +995,18 @@ fn indent_string_part(&mut self, range: Range<usize>, is_double_quoted: bool) {
impl<'a> NodeVisitor<'a> for IndentVisitor<'a> {
// Default implementation is to just visit children.
fn visit(&mut self, node: &'a dyn Node) {
let mut inc = 0;
let mut dec = 0;
use ast::{Category, Type};
match node.typ() {
Type::job_list | Type::andor_job_list => {
let mut inc_dec = (0, 0);
match node.kind() {
Kind::JobList(_) | Kind::AndorJobList(_) => {
// Job lists are never unwound.
inc = 1;
dec = 1;
inc_dec = (1, 1);
}
// Increment indents for conditions in headers (#1665).
Type::job_conjunction => {
let typ = self.parent.unwrap().typ();
if matches!(typ, Type::if_clause | Type::while_header) {
inc = 1;
dec = 1;
Kind::JobConjunction(_node) => {
let parent_kind = self.parent.unwrap().kind();
if matches!(parent_kind, Kind::IfClause(_) | Kind::WhileHeader(_)) {
inc_dec = (1, 1);
}
}
@@ -1023,22 +1019,20 @@ fn visit(&mut self, node: &'a dyn Node) {
// ....cmd3
// end
// See #7252.
Type::job_continuation => {
if self.has_newline(&node.as_job_continuation().unwrap().newlines) {
inc = 1;
dec = 1;
Kind::JobContinuation(node) => {
if self.has_newline(&node.newlines) {
inc_dec = (1, 1);
}
}
// Likewise for && and ||.
Type::job_conjunction_continuation => {
if self.has_newline(&node.as_job_conjunction_continuation().unwrap().newlines) {
inc = 1;
dec = 1;
Kind::JobConjunctionContinuation(node) => {
if self.has_newline(&node.newlines) {
inc_dec = (1, 1);
}
}
Type::case_item_list => {
Kind::CaseItemList(_) => {
// Here's a hack. Consider:
// switch abc
// cas
@@ -1056,37 +1050,35 @@ fn visit(&mut self, node: &'a dyn Node) {
// And so we will think that the 'cas' job is at the same level as the switch.
// To address this, if we see that the switch statement was not closed, do not
// decrement the indent afterwards.
inc = 1;
let switchs = self.parent.unwrap().as_switch_statement().unwrap();
dec = if switchs.end.has_source() { 1 } else { 0 };
let Kind::SwitchStatement(switchs) = self.parent.unwrap().kind() else {
panic!("Expected switch statement");
};
let dec = if switchs.end.has_source() { 1 } else { 0 };
inc_dec = (1, dec);
}
Type::token_base => {
let token_type = node.as_token().unwrap().token_type();
let parent_type = self.parent.unwrap().typ();
if parent_type == Type::begin_header && token_type == ParseTokenType::end {
Kind::Token(node) => {
let token_type = node.token_type();
let parent_kind = self.parent.unwrap().kind();
if matches!(parent_kind, Kind::BeginHeader(_)) && token_type == ParseTokenType::end
{
// The newline after "begin" is optional, so it is part of the header.
// The header is not in the indented block, so indent the newline here.
if node.source(self.src) == "\n" {
inc = 1;
dec = 1;
inc_dec = (1, 1);
}
}
// if token_type == ParseTokenType::right_brace && parent_type == Type::brace_statement
// {
// inc = 1;
// dec = 1;
// }
}
_ => (),
}
_ => {}
}
let range = node.source_range();
if range.length() > 0 && node.category() == Category::leaf {
if range.length() > 0 && node.as_leaf().is_some() {
self.record_line_continuations_until(range.start());
self.indents[self.last_leaf_end..range.start()].fill(self.last_indent);
}
self.indent += inc;
self.indent += inc_dec.0;
// If we increased the indentation, apply it to the remainder of the string, even if the
// list is empty. For example (where _ represents the cursor):
@@ -1095,12 +1087,12 @@ fn visit(&mut self, node: &'a dyn Node) {
// _
//
// we want to indent the newline.
if inc != 0 {
if inc_dec.0 != 0 {
self.last_indent = self.indent;
}
// If this is a leaf node, apply the current indentation.
if node.category() == Category::leaf && range.length() != 0 {
if node.as_leaf().is_some() && range.length() != 0 {
let leading_spaces = self.src[..range.start()]
.chars()
.rev()
@@ -1113,9 +1105,9 @@ fn visit(&mut self, node: &'a dyn Node) {
}
let saved = self.parent.replace(node);
node.accept(self, false);
node.accept(self);
self.parent = saved;
self.indent -= dec;
self.indent -= inc_dec.1;
}
}
@@ -1140,7 +1132,7 @@ pub fn parse_util_detect_errors(
// Parse the input string into an ast. Some errors are detected here.
let mut parse_errors = ParseErrorList::new();
let ast = Ast::parse(buff_src, parse_flags, Some(&mut parse_errors));
let ast = ast::parse(buff_src, parse_flags, Some(&mut parse_errors));
if allow_incomplete {
// Issue #1238: If the only error was unterminated quote, then consider this to have parsed
// successfully.
@@ -1210,76 +1202,95 @@ pub fn parse_util_detect_errors_in_ast(
let mut traversal = ast::Traversal::new(ast.top());
while let Some(node) = traversal.next() {
if let Some(jc) = node.as_job_continuation() {
// Somewhat clumsy way of checking for a statement without source in a pipeline.
// See if our pipe has source but our statement does not.
if jc.pipe.has_source() && jc.statement.try_source_range().is_none() {
has_unclosed_pipe = true;
match node.kind() {
Kind::JobContinuation(jc) => {
// Somewhat clumsy way of checking for a statement without source in a pipeline.
// See if our pipe has source but our statement does not.
if jc.pipe.has_source() && jc.statement.try_source_range().is_none() {
has_unclosed_pipe = true;
}
}
} else if let Some(job_conjunction) = node.as_job_conjunction() {
errored |= detect_errors_in_job_conjunction(job_conjunction, &mut out_errors);
} else if let Some(jcc) = node.as_job_conjunction_continuation() {
// Somewhat clumsy way of checking for a job without source in a conjunction.
// See if our conjunction operator (&& or ||) has source but our job does not.
if jcc.conjunction.has_source() && jcc.job.try_source_range().is_none() {
has_unclosed_conjunction = true;
Kind::JobConjunction(job_conjunction) => {
errored |= detect_errors_in_job_conjunction(job_conjunction, &mut out_errors);
}
} else if let Some(arg) = node.as_argument() {
let arg_src = arg.source(buff_src);
res |= parse_util_detect_errors_in_argument(arg, arg_src, &mut out_errors)
.err()
.unwrap_or_default();
} else if let Some(job) = node.as_job_pipeline() {
// Disallow background in the following cases:
//
// foo & ; and bar
// foo & ; or bar
// if foo & ; end
// while foo & ; end
// If it's not a background job, nothing to do.
if job.bg.is_some() {
errored |= detect_errors_in_backgrounded_job(&traversal, job, &mut out_errors);
Kind::JobConjunctionContinuation(jcc) => {
// Somewhat clumsy way of checking for a job without source in a conjunction.
// See if our conjunction operator (&& or ||) has source but our job does not.
if jcc.conjunction.has_source() && jcc.job.try_source_range().is_none() {
has_unclosed_conjunction = true;
}
}
} else if let Some(stmt) = node.as_decorated_statement() {
errored |=
detect_errors_in_decorated_statement(buff_src, &traversal, stmt, &mut out_errors);
} else if let Some(block) = node.as_block_statement() {
// If our 'end' had no source, we are unsourced.
if !block.end.has_source() {
has_unclosed_block = true;
Kind::Argument(arg) => {
let arg_src = arg.source(buff_src);
res |= parse_util_detect_errors_in_argument(arg, arg_src, &mut out_errors)
.err()
.unwrap_or_default();
}
errored |= detect_errors_in_block_redirection_list(
node,
&block.args_or_redirs,
&mut out_errors,
);
} else if let Some(brace_statement) = node.as_brace_statement() {
// If our closing brace had no source, we are unsourced.
if !brace_statement.right_brace.has_source() {
has_unclosed_block = true;
Kind::JobPipeline(job) => {
// Disallow background in the following cases:
//
// foo & ; and bar
// foo & ; or bar
// if foo & ; end
// while foo & ; end
// If it's not a background job, nothing to do.
if job.bg.is_some() {
errored |= detect_errors_in_backgrounded_job(&traversal, job, &mut out_errors);
}
}
errored |= detect_errors_in_block_redirection_list(
node,
&brace_statement.args_or_redirs,
&mut out_errors,
);
} else if let Some(ifs) = node.as_if_statement() {
// If our 'end' had no source, we are unsourced.
if !ifs.end.has_source() {
has_unclosed_block = true;
Kind::DecoratedStatement(stmt) => {
errored |= detect_errors_in_decorated_statement(
buff_src,
&traversal,
stmt,
&mut out_errors,
);
}
errored |=
detect_errors_in_block_redirection_list(node, &ifs.args_or_redirs, &mut out_errors);
} else if let Some(switchs) = node.as_switch_statement() {
// If our 'end' had no source, we are unsourced.
if !switchs.end.has_source() {
has_unclosed_block = true;
Kind::BlockStatement(block) => {
// If our 'end' had no source, we are unsourced.
if !block.end.has_source() {
has_unclosed_block = true;
}
errored |= detect_errors_in_block_redirection_list(
node,
&block.args_or_redirs,
&mut out_errors,
);
}
errored |= detect_errors_in_block_redirection_list(
node,
&switchs.args_or_redirs,
&mut out_errors,
);
Kind::BraceStatement(brace_statement) => {
// If our closing brace had no source, we are unsourced.
if !brace_statement.right_brace.has_source() {
has_unclosed_block = true;
}
errored |= detect_errors_in_block_redirection_list(
node,
&brace_statement.args_or_redirs,
&mut out_errors,
);
}
Kind::IfStatement(ifs) => {
// If our 'end' had no source, we are unsourced.
if !ifs.end.has_source() {
has_unclosed_block = true;
}
errored |= detect_errors_in_block_redirection_list(
node,
&ifs.args_or_redirs,
&mut out_errors,
);
}
Kind::SwitchStatement(switchs) => {
// If our 'end' had no source, we are unsourced.
if !switchs.end.has_source() {
has_unclosed_block = true;
}
errored |= detect_errors_in_block_redirection_list(
node,
&switchs.args_or_redirs,
&mut out_errors,
);
}
_ => {}
}
}
@@ -1316,14 +1327,15 @@ pub fn parse_util_detect_errors_in_argument_list(
// Parse the string as a freestanding argument list.
let mut errors = ParseErrorList::new();
let ast = Ast::parse_argument_list(arg_list_src, ParseTreeFlags::empty(), Some(&mut errors));
let ast = ast::parse_argument_list(arg_list_src, ParseTreeFlags::empty(), Some(&mut errors));
if !errors.is_empty() {
return get_error_text(&errors);
}
// Get the root argument list and extract arguments from it.
// Test each of these.
let args = &ast.top().as_freestanding_argument_list().unwrap().arguments;
let arg_list: &ast::FreestandingArgumentList = ast.top();
let args = &arg_list.arguments;
for arg in args.iter() {
let arg_src = arg.source(arg_list_src);
if parse_util_detect_errors_in_argument(arg, arg_src, &mut Some(&mut errors)).is_err() {
@@ -1553,19 +1565,22 @@ fn detect_errors_in_backgrounded_job(
// foo & ; or bar
// if foo & ; end
// while foo & ; end
let Some(job_conj) = traversal.parent(job).as_job_conjunction() else {
let Kind::JobConjunction(job_conj) = traversal.parent(job).kind() else {
return false;
};
let job_conj_parent = traversal.parent(job_conj);
if job_conj_parent.as_if_clause().is_some() || job_conj_parent.as_while_header().is_some() {
if matches!(
job_conj_parent.kind(),
Kind::IfClause(_) | Kind::WhileHeader(_)
) {
errored = append_syntax_error!(
parse_errors,
source_range.start(),
source_range.length(),
BACKGROUND_IN_CONDITIONAL_ERROR_MSG
);
} else if let Some(jlist) = job_conj_parent.as_job_list() {
} else if let Kind::JobList(jlist) = job_conj_parent.kind() {
// This isn't very complete, e.g. we don't catch 'foo & ; not and bar'.
// Find the index of ourselves in the job list.
let index = jlist
@@ -1619,13 +1634,18 @@ fn detect_errors_in_decorated_statement(
}
// Get the statement we are part of.
let st = traversal.parent(dst).as_statement().unwrap();
let Kind::Statement(st) = traversal.parent(dst).kind() else {
panic!();
};
// Walk up to the job.
let job = traversal
.parent_nodes()
.find_map(|n| n.as_job_pipeline())
.expect("Should have found the job");
.find_map(|n| match n.kind() {
Kind::JobPipeline(job) => Some(job),
_ => None,
})
.expect("should have found the job");
// Check our pipeline position.
let pipe_pos = if job.continuation.is_empty() {
@@ -1736,17 +1756,17 @@ fn detect_errors_in_decorated_statement(
// loop from the ancestor alone; we need the header. That is, we hit a
// block_statement, and have to check its header.
let mut found_loop = false;
for block in traversal
.parent_nodes()
.filter_map(|anc| anc.as_block_statement())
{
match block.header.typ() {
ast::Type::for_header | ast::Type::while_header => {
for block in traversal.parent_nodes().filter_map(|anc| match anc.kind() {
Kind::BlockStatement(block) => Some(block),
_ => None,
}) {
match block.header {
ast::BlockStatementHeader::For(_) | ast::BlockStatementHeader::While(_) => {
// This is a loop header, so we can break or continue.
found_loop = true;
break;
}
ast::Type::function_header => {
ast::BlockStatementHeader::Function(_) => {
// This is a function header, so we cannot break or
// continue. We stop our search here.
found_loop = false;
@@ -1820,7 +1840,7 @@ fn detect_errors_in_block_redirection_list(
return false;
};
let r = first_arg.source_range();
if parent.as_brace_statement().is_some() {
if let Kind::BraceStatement(_) = parent.kind() {
append_syntax_error!(out_errors, r.start(), r.length(), RIGHT_BRACE_ARG_ERR_MSG);
} else {
append_syntax_error!(out_errors, r.start(), r.length(), END_ARG_ERR_MSG);

11
src/parser.rs
View File

@@ -1,6 +1,6 @@
// The fish parser. Contains functions for parsing and evaluating code.
use crate::ast::{self, Ast, List, Node};
use crate::ast::{self, Node};
use crate::builtins::shared::STATUS_ILLEGAL_CMD;
use crate::common::{
escape_string, wcs2string, CancelChecker, EscapeFlags, EscapeStringStyle, FilenameRef,
@@ -556,7 +556,7 @@ pub fn eval_parsed_source(
block_type: BlockType,
) -> EvalRes {
assert!([BlockType::top, BlockType::subst].contains(&block_type));
let job_list = ps.ast.top().as_job_list().unwrap();
let job_list = ps.ast.top();
if !job_list.is_empty() {
// Execute the top job list.
self.eval_node(ps, job_list, io, job_group, block_type)
@@ -581,7 +581,7 @@ pub fn eval_wstr(
use crate::parse_tree::ParsedSource;
use crate::parse_util::parse_util_detect_errors_in_ast;
let mut errors = vec![];
let ast = Ast::parse(&src, ParseTreeFlags::empty(), Some(&mut errors));
let ast = ast::parse(&src, ParseTreeFlags::empty(), Some(&mut errors));
let mut errored = ast.errored();
if !errored {
errored = parse_util_detect_errors_in_ast(&ast, &src, Some(&mut errors)).is_err();
@@ -726,7 +726,7 @@ pub fn expand_argument_list(
ctx: &OperationContext<'_>,
) -> CompletionList {
// Parse the string as an argument list.
let ast = Ast::parse_argument_list(arg_list_src, ParseTreeFlags::default(), None);
let ast = ast::parse_argument_list(arg_list_src, ParseTreeFlags::default(), None);
if ast.errored() {
// Failed to parse. Here we expect to have reported any errors in test_args.
return vec![];
@@ -734,8 +734,7 @@ pub fn expand_argument_list(
// Get the root argument list and extract arguments from it.
let mut result = vec![];
let list = ast.top().as_freestanding_argument_list().unwrap();
for arg in &list.arguments {
for arg in &ast.top().arguments {
let arg_src = arg.source(arg_list_src);
if matches!(
expand_string(arg_src.to_owned(), &mut result, flags, ctx, None).result,

16
src/reader.rs
View File

@@ -45,7 +45,7 @@
use errno::{errno, Errno};
use crate::abbrs::abbrs_match;
use crate::ast::{is_same_node, Ast, Category};
use crate::ast::{self, is_same_node, Kind};
use crate::builtins::shared::ErrorCode;
use crate::builtins::shared::STATUS_CMD_ERROR;
use crate::builtins::shared::STATUS_CMD_OK;
@@ -5339,15 +5339,15 @@ fn extract_tokens(s: &wstr) -> Vec<PositionedToken> {
let ast_flags = ParseTreeFlags::CONTINUE_AFTER_ERROR
| ParseTreeFlags::ACCEPT_INCOMPLETE_TOKENS
| ParseTreeFlags::LEAVE_UNTERMINATED;
let ast = Ast::parse(s, ast_flags, None);
let ast = ast::parse(s, ast_flags, None);
let mut result = vec![];
let mut traversal = ast.walk();
while let Some(node) = traversal.next() {
// We are only interested in leaf nodes with source.
if node.category() != Category::leaf {
if node.as_leaf().is_none() {
continue;
}
};
let range = node.source_range();
if range.length() == 0 {
continue;
@@ -5381,10 +5381,10 @@ fn extract_tokens(s: &wstr) -> Vec<PositionedToken> {
if !has_cmd_subs {
// Common case of no command substitutions in this leaf node.
// Check if a node is the command portion of a decorated statement.
let is_cmd = traversal
.parent(node)
.as_decorated_statement()
.is_some_and(|stmt| is_same_node(node, &stmt.command));
let mut is_cmd = false;
if let Kind::DecoratedStatement(stmt) = traversal.parent(node).kind() {
is_cmd = is_same_node(node, &stmt.command);
}
result.push(PositionedToken { range, is_cmd })
}
}

4
src/tests/ast.rs
View File

@@ -1,4 +1,4 @@
use crate::ast::{is_same_node, Ast, Node};
use crate::ast::{self, is_same_node, Node};
use crate::wchar::prelude::*;
const FISH_FUNC: &str = r#"
@@ -30,7 +30,7 @@
fn test_is_same_node() {
// is_same_node is pretty subtle! Let's check it.
let src = WString::from_str(FISH_FUNC);
let ast = Ast::parse(&src, Default::default(), None);
let ast = ast::parse(&src, Default::default(), None);
assert!(!ast.errored());
let all_nodes: Vec<&dyn Node> = ast.walk().collect();
for i in 0..all_nodes.len() {

4
src/tests/ast_bench.rs
View File

@@ -2,7 +2,7 @@
#[cfg(feature = "benchmark")]
mod bench {
extern crate test;
use crate::ast::Ast;
use crate::ast;
use crate::wchar::prelude::*;
use test::Bencher;
@@ -39,7 +39,7 @@ fn bench_ast_construction(b: &mut Bencher) {
let src = generate_fish_script();
b.bytes = (src.len() * 4) as u64; // 4 bytes per character
b.iter(|| {
let _ast = Ast::parse(&src, Default::default(), None);
let _ast = ast::parse(&src, Default::default(), None);
});
}
}

64
src/tests/parser.rs
View File

@@ -1,4 +1,4 @@
use crate::ast::{self, is_same_node, Ast, JobPipeline, List, Node, Traversal};
use crate::ast::{self, is_same_node, Ast, Castable, JobList, JobPipeline, Kind, Node, Traversal};
use crate::common::ScopeGuard;
use crate::env::EnvStack;
use crate::expand::ExpandFlags;
@@ -31,11 +31,11 @@ macro_rules! detect_errors {
fn detect_argument_errors(src: &str) -> Result<(), ParserTestErrorBits> {
let src = WString::from_str(src);
let ast = Ast::parse_argument_list(&src, ParseTreeFlags::default(), None);
let ast = ast::parse_argument_list(&src, ParseTreeFlags::default(), None);
if ast.errored() {
return Err(ParserTestErrorBits::ERROR);
}
let args = &ast.top().as_freestanding_argument_list().unwrap().arguments;
let args = &ast.top().arguments;
let first_arg = args.get(0).expect("Failed to parse an argument");
let mut errors = None;
parse_util_detect_errors_in_argument(first_arg, first_arg.source(&src), &mut errors)
@@ -323,7 +323,7 @@ fn test_new_parser_correctness() {
let _cleanup = test_init();
macro_rules! validate {
($src:expr, $ok:expr) => {
let ast = Ast::parse(L!($src), ParseTreeFlags::default(), None);
let ast = ast::parse(L!($src), ParseTreeFlags::default(), None);
assert_eq!(ast.errored(), !$ok);
};
}
@@ -400,7 +400,7 @@ fn string_for_permutation(fuzzes: &[&wstr], len: usize, permutation: usize) -> O
let mut permutation = 0;
while let Some(src) = string_for_permutation(&fuzzes, len, permutation) {
permutation += 1;
Ast::parse(&src, ParseTreeFlags::default(), None);
ast::parse(&src, ParseTreeFlags::default(), None);
}
}
}
@@ -416,7 +416,7 @@ fn test_new_parser_ll2() {
// Parse a statement, returning the command, args (joined by spaces), and the decoration. Returns
// true if successful.
fn test_1_parse_ll2(src: &wstr) -> Option<(WString, WString, StatementDecoration)> {
let ast = Ast::parse(src, ParseTreeFlags::default(), None);
let ast = ast::parse(src, ParseTreeFlags::default(), None);
if ast.errored() {
return None;
}
@@ -424,7 +424,7 @@ fn test_1_parse_ll2(src: &wstr) -> Option<(WString, WString, StatementDecoration
// Get the statement. Should only have one.
let mut statement = None;
for n in Traversal::new(ast.top()) {
if let Some(tmp) = n.as_decorated_statement() {
if let Kind::DecoratedStatement(tmp) = n.kind() {
assert!(
statement.is_none(),
"More than one decorated statement found in '{}'",
@@ -513,21 +513,21 @@ macro_rules! validate {
// Verify that 'function -h' and 'function --help' are plain statements but 'function --foo' is
// not (issue #1240).
macro_rules! check_function_help {
($src:expr, $typ:expr) => {
let ast = Ast::parse(L!($src), ParseTreeFlags::default(), None);
($src:expr, $kind:pat) => {
let ast = ast::parse(L!($src), ParseTreeFlags::default(), None);
assert!(!ast.errored());
assert_eq!(
Traversal::new(ast.top())
.filter(|n| n.typ() == $typ)
.filter(|n| matches!(n.kind(), $kind))
.count(),
1
);
};
}
check_function_help!("function -h", ast::Type::decorated_statement);
check_function_help!("function --help", ast::Type::decorated_statement);
check_function_help!("function --foo; end", ast::Type::function_header);
check_function_help!("function foo; end", ast::Type::function_header);
check_function_help!("function -h", ast::Kind::DecoratedStatement(_));
check_function_help!("function --help", ast::Kind::DecoratedStatement(_));
check_function_help!("function --foo; end", ast::Kind::FunctionHeader(_));
check_function_help!("function foo; end", ast::Kind::FunctionHeader(_));
}
#[test]
@@ -538,13 +538,13 @@ fn test_new_parser_ad_hoc() {
// Ensure that 'case' terminates a job list.
let src = L!("switch foo ; case bar; case baz; end");
let ast = Ast::parse(src, ParseTreeFlags::default(), None);
let ast = ast::parse(src, ParseTreeFlags::default(), None);
assert!(!ast.errored());
// Expect two case_item_lists. The bug was that we'd
// Expect two CaseItems. The bug was that we'd
// try to run a command 'case'.
assert_eq!(
Traversal::new(ast.top())
.filter(|n| n.typ() == ast::Type::case_item)
.filter(|n| matches!(n.kind(), ast::Kind::CaseItem(_)))
.count(),
2
);
@@ -554,17 +554,17 @@ fn test_new_parser_ad_hoc() {
// leading to an infinite loop.
// By itself it should produce an error.
let ast = Ast::parse(L!("a="), ParseTreeFlags::default(), None);
let ast = ast::parse(L!("a="), ParseTreeFlags::default(), None);
assert!(ast.errored());
// If we are leaving things unterminated, this should not produce an error.
// i.e. when typing "a=" at the command line, it should be treated as valid
// because we don't want to color it as an error.
let ast = Ast::parse(L!("a="), ParseTreeFlags::LEAVE_UNTERMINATED, None);
let ast = ast::parse(L!("a="), ParseTreeFlags::LEAVE_UNTERMINATED, None);
assert!(!ast.errored());
let mut errors = vec![];
Ast::parse(
ast::parse(
L!("begin; echo ("),
ParseTreeFlags::LEAVE_UNTERMINATED,
Some(&mut errors),
@@ -573,7 +573,7 @@ fn test_new_parser_ad_hoc() {
assert!(errors[0].code == ParseErrorCode::tokenizer_unterminated_subshell);
errors.clear();
Ast::parse(
ast::parse(
L!("for x in ("),
ParseTreeFlags::LEAVE_UNTERMINATED,
Some(&mut errors),
@@ -582,7 +582,7 @@ fn test_new_parser_ad_hoc() {
assert!(errors[0].code == ParseErrorCode::tokenizer_unterminated_subshell);
errors.clear();
Ast::parse(
ast::parse(
L!("begin; echo '"),
ParseTreeFlags::LEAVE_UNTERMINATED,
Some(&mut errors),
@@ -598,7 +598,7 @@ fn test_new_parser_errors() {
macro_rules! validate {
($src:expr, $expected_code:expr) => {
let mut errors = vec![];
let ast = Ast::parse(L!($src), ParseTreeFlags::default(), Some(&mut errors));
let ast = ast::parse(L!($src), ParseTreeFlags::default(), Some(&mut errors));
assert!(ast.errored());
assert_eq!(
errors.into_iter().map(|e| e.code).collect::<Vec<_>>(),
@@ -786,7 +786,7 @@ fn test_line_counter() {
assert_eq!(line_offset, expected);
}
let pipelines: Vec<_> = ps.ast.walk().filter_map(|n| n.as_job_pipeline()).collect();
let pipelines: Vec<_> = ps.ast.walk().filter_map(ast::JobPipeline::cast).collect();
assert_eq!(pipelines.len(), 3);
let src_offsets = [0, 0, 2];
assert_eq!(line_counter.source_offset_of_node(), None);
@@ -832,14 +832,13 @@ struct TrueSemiAstTester<'a> {
impl<'a> TrueSemiAstTester<'a> {
const TRUE_SEMI: &'static wstr = L!("true;");
fn new(ast: &'a Ast) -> Self {
let job_list = ast.top().as_job_list().expect("Expected job_list");
let job_list: &JobList = ast.top();
let job_conjunction = &job_list[0];
let job_pipeline = &job_conjunction.job;
let variable_assignment_list = &job_pipeline.variables;
let statement = &job_pipeline.statement;
let decorated_statement = statement
.contents
.as_decorated_statement()
.expect("Expected decorated_statement");
let command = &decorated_statement.command;
@@ -893,9 +892,8 @@ fn get_parents<'s>(&'s self, node: &'a dyn Node) -> impl Iterator<Item = &'a dyn
#[test]
fn test_ast() {
// Light testing of the AST and traversals.
let ast = Ast::parse(TrueSemiAstTester::TRUE_SEMI, ParseTreeFlags::empty(), None);
let ast = ast::parse(TrueSemiAstTester::TRUE_SEMI, ParseTreeFlags::empty(), None);
let tester = TrueSemiAstTester::new(&ast);
assert!(ast.top().as_job_list().is_some(), "Expected job_list");
// Walk the AST and collect all nodes.
// See is_same_node comments for why we can't use assert_eq! here.
@@ -920,7 +918,7 @@ fn test_ast() {
// Find the decorated statement.
let decorated_statement = ast
.walk()
.find(|n| n.typ() == ast::Type::decorated_statement)
.find(|n| matches!(n.kind(), ast::Kind::DecoratedStatement(_)))
.expect("Expected decorated statement");
// Test the skip feature. Don't descend into the decorated_statement.
@@ -955,10 +953,10 @@ fn test_ast() {
#[should_panic]
fn test_traversal_skip_children_panics() {
// Test that we panic if we try to skip children of a node that is not the current node.
let ast = Ast::parse(L!("true;"), ParseTreeFlags::empty(), None);
let ast = ast::parse(L!("true;"), ParseTreeFlags::empty(), None);
let mut traversal = ast.walk();
while let Some(node) = traversal.next() {
if node.typ() == ast::Type::decorated_statement {
if matches!(node.kind(), ast::Kind::DecoratedStatement(_)) {
// Should panic as we can only skip the current node.
traversal.skip_children(ast.top());
}
@@ -969,11 +967,11 @@ fn test_traversal_skip_children_panics() {
#[should_panic]
fn test_traversal_parent_panics() {
// Can only get the parent of nodes still on the stack.
let ast = Ast::parse(L!("true;"), ParseTreeFlags::empty(), None);
let ast = ast::parse(L!("true;"), ParseTreeFlags::empty(), None);
let mut traversal = ast.walk();
let mut decorated_statement = None;
while let Some(node) = traversal.next() {
if node.as_decorated_statement().is_some() {
if let Kind::DecoratedStatement(_) = node.kind() {
decorated_statement = Some(node);
} else if node.as_token().map(|t| t.token_type()) == Some(ParseTokenType::end) {
// should panic as the decorated_statement is not on the stack.