refactor: move char definitions into widestring

Use `fish_widestring` as the place where char definitions live. This has
the advantage that all our code can depend on `fish_widestring` without
introducing dependency cycles. Having a common place for character
definitions also makes it easier to see which chars have a special
meaning assigned to them.

This change also unblocks some follow-up refactoring by removing a
dependency cycle between `src/common.rs` and `src/wildcard.rs`.

Part of #12625

src/builtins/string/match.rs

@@ -8,7 +8,8 @@
 use crate::flog::flog;
 use crate::parse_util::unescape_wildcards;
 use crate::parser::ParserEnvSetMode;
-use crate::wildcard::{ANY_STRING, wildcard_match};
+use crate::wildcard::wildcard_match;
+use fish_widestring::ANY_STRING;
 
 #[derive(Default)]
 pub struct Match<'args> {

src/common.rs

@@ -9,12 +9,15 @@
     prelude::*,
     terminal::Outputter,
     termsize::Termsize,
-    wildcard::{ANY_CHAR, ANY_STRING, ANY_STRING_RECURSIVE},
     wutil::fish_iswalnum,
 };
 use fish_fallback::fish_wcwidth;
 use fish_feature_flags::{FeatureFlag, feature_test};
-use fish_widestring::{decode_byte_from_char, encode_byte_to_char, subslice_position, wcs2bytes};
+use fish_widestring::{
+    ANY_CHAR, ANY_STRING, ANY_STRING_RECURSIVE, ASCII_MAX, BYTE_MAX, UCS2_MAX,
+    decode_byte_from_char, encode_byte_to_char, fish_reserved_codepoint, subslice_position,
+    wcs2bytes,
+};
 use nix::sys::termios::Termios;
 use std::{
     env,

src/expand.rs

@@ -3,35 +3,37 @@
 //! from using a more clever memory allocation scheme, perhaps an evil combination of talloc,
 //! string buffers and reference counting.
 
-use crate::builtins::shared::{
-    STATUS_CMD_ERROR, STATUS_CMD_UNKNOWN, STATUS_EXPAND_ERROR, STATUS_ILLEGAL_CMD,
-    STATUS_INVALID_ARGS, STATUS_NOT_EXECUTABLE, STATUS_READ_TOO_MUCH, STATUS_UNMATCHED_WILDCARD,
+use crate::{
+    builtins::shared::{
+        STATUS_CMD_ERROR, STATUS_CMD_UNKNOWN, STATUS_EXPAND_ERROR, STATUS_ILLEGAL_CMD,
+        STATUS_INVALID_ARGS, STATUS_NOT_EXECUTABLE, STATUS_READ_TOO_MUCH,
+        STATUS_UNMATCHED_WILDCARD,
+    },
+    common::{
+        escape, escape_string, escape_string_for_double_quotes, osstr2wcstring, unescape_string,
+        valid_var_name_char,
+    },
+    complete::{CompleteFlags, Completion, CompletionList, CompletionReceiver},
+    env::{EnvVar, Environment},
+    exec::exec_subshell_for_expand,
+    history::{History, history_session_id},
+    operation_context::OperationContext,
+    parse_constants::{ParseError, ParseErrorCode, ParseErrorList, SOURCE_LOCATION_UNKNOWN},
+    parse_util::{MaybeParentheses, expand_variable_error, locate_cmdsubst_range},
+    path::path_apply_working_directory,
+    prelude::*,
+    wildcard::{WildcardResult, wildcard_expand_string, wildcard_has_internal},
+    wutil::{Options, normalize_path, wcstoi_partial},
 };
-use crate::common::{
-    escape, escape_string, escape_string_for_double_quotes, osstr2wcstring, unescape_string,
-    valid_var_name_char,
-};
-use crate::complete::{CompleteFlags, Completion, CompletionList, CompletionReceiver};
-use crate::env::{EnvVar, Environment};
-use crate::exec::exec_subshell_for_expand;
-use crate::history::{History, history_session_id};
-use crate::operation_context::OperationContext;
-use crate::parse_constants::{ParseError, ParseErrorCode, ParseErrorList, SOURCE_LOCATION_UNKNOWN};
-use crate::parse_util::{MaybeParentheses, expand_variable_error, locate_cmdsubst_range};
-use crate::path::path_apply_working_directory;
-use crate::prelude::*;
-use crate::wildcard::{ANY_CHAR, ANY_STRING, ANY_STRING_RECURSIVE, WildcardResult};
-use crate::wildcard::{wildcard_expand_string, wildcard_has_internal};
-use crate::wutil::{Options, normalize_path, wcstoi_partial};
 use bitflags::bitflags;
-use fish_common::{
-    EXPAND_RESERVED_BASE, EXPAND_RESERVED_END, EscapeFlags, EscapeStringStyle, UnescapeFlags,
-    UnescapeStringStyle,
-};
+use fish_common::{EscapeFlags, EscapeStringStyle, UnescapeFlags, UnescapeStringStyle};
 use fish_feature_flags::{FeatureFlag, feature_test};
 use fish_util::wcsfilecmp_glob;
 use fish_wcstringutil::{join_strings, trim};
-use fish_widestring::char_offset;
+use fish_widestring::{
+    ANY_CHAR, ANY_STRING, ANY_STRING_RECURSIVE, EXPAND_RESERVED_BASE, EXPAND_RESERVED_END,
+    char_offset,
+};
 use nix::unistd::{User, getpid};
 
 bitflags! {
@@ -1576,25 +1578,20 @@ pub struct ExpandResult {
 
 #[cfg(test)]
 mod tests {
-    use crate::abbrs::Abbreviation;
-    use crate::abbrs::{self};
-    use crate::abbrs::{with_abbrs, with_abbrs_mut};
-    use crate::common::str2wcstring;
-    use crate::complete::{CompletionList, CompletionReceiver};
-    use crate::env::{EnvMode, EnvStackSetResult};
-    use crate::expand::{ExpandResultCode, expand_to_receiver};
-    use crate::operation_context::{EXPANSION_LIMIT_DEFAULT, no_cancel};
-    use crate::parse_constants::ParseErrorList;
-    use crate::parser::ParserEnvSetMode;
-    use crate::tests::prelude::*;
-    use crate::wildcard::ANY_STRING;
     use crate::{
-        expand::{ExpandFlags, expand_string},
-        operation_context::OperationContext,
+        abbrs::{self, Abbreviation, with_abbrs, with_abbrs_mut},
+        common::str2wcstring,
+        complete::{CompletionList, CompletionReceiver},
+        env::{EnvMode, EnvStackSetResult},
+        expand::{ExpandFlags, ExpandResultCode, expand_string, expand_to_receiver},
+        operation_context::{EXPANSION_LIMIT_DEFAULT, OperationContext, no_cancel},
+        parse_constants::ParseErrorList,
+        parser::ParserEnvSetMode,
         prelude::*,
+        tests::prelude::*,
     };
-    use std::collections::HashSet;
-    use std::collections::hash_map::RandomState;
+    use fish_widestring::ANY_STRING;
+    use std::collections::{HashSet, hash_map::RandomState};
 
     fn expand_test_impl(
         input: &wstr,

src/highlight/file_tester.rs

@@ -3,24 +3,22 @@
 // Because this may perform blocking I/O, we compute results in a separate thread,
 // and provide them optimistically.
 use crate::common::unescape_string;
-use crate::expand::{
-    BRACE_BEGIN, BRACE_END, BRACE_SEP, INTERNAL_SEPARATOR, PROCESS_EXPAND_SELF, VARIABLE_EXPAND,
-    VARIABLE_EXPAND_SINGLE, expand_one,
-};
-use crate::expand::{ExpandFlags, HOME_DIRECTORY, expand_tilde};
-use crate::operation_context::OperationContext;
-use crate::path::path_apply_working_directory;
-use crate::redirection::RedirectionMode;
-use crate::threads::assert_is_background_thread;
-use crate::wildcard::{ANY_CHAR, ANY_STRING, ANY_STRING_RECURSIVE};
-use crate::wutil::{
-    dir_iter::DirIter, fish_wcstoi, normalize_path, waccess, wbasename, wdirname, wstat,
+use crate::{
+    expand::{
+        BRACE_BEGIN, BRACE_END, BRACE_SEP, ExpandFlags, HOME_DIRECTORY, INTERNAL_SEPARATOR,
+        PROCESS_EXPAND_SELF, VARIABLE_EXPAND, VARIABLE_EXPAND_SINGLE, expand_one, expand_tilde,
+    },
+    operation_context::OperationContext,
+    path::path_apply_working_directory,
+    redirection::RedirectionMode,
+    threads::assert_is_background_thread,
+    wutil::{dir_iter::DirIter, fish_wcstoi, normalize_path, waccess, wbasename, wdirname, wstat},
 };
 use fish_common::{UnescapeFlags, UnescapeStringStyle};
 use fish_wcstringutil::{
     string_prefixes_string, string_prefixes_string_case_insensitive, string_suffixes_string,
 };
-use fish_widestring::{L, WExt as _, WString, wstr};
+use fish_widestring::{ANY_CHAR, ANY_STRING, ANY_STRING_RECURSIVE, L, WExt as _, WString, wstr};
 use libc::PATH_MAX;
 use nix::unistd::AccessFlags;
 use std::collections::{HashMap, HashSet};

src/highlight/highlight.rs

@@ -1,40 +1,41 @@
 //! Functions for syntax highlighting.
-use crate::abbrs::{self, with_abbrs};
-use crate::ast::{
-    self, Argument, BlockStatement, BlockStatementHeader, BraceStatement, DecoratedStatement,
-    Keyword, Kind, Node, NodeVisitor, Redirection, Token, VariableAssignment,
+use crate::{
+    abbrs::{self, with_abbrs},
+    ast::{
+        self, Argument, BlockStatement, BlockStatementHeader, BraceStatement, DecoratedStatement,
+        Keyword, Kind, Node, NodeVisitor, Redirection, Token, VariableAssignment,
+    },
+    builtins::shared::builtin_exists,
+    common::{valid_var_name, valid_var_name_char},
+    complete::complete_wrap_map,
+    env::{EnvVar, Environment},
+    expand::{
+        ExpandFlags, ExpandResultCode, PROCESS_EXPAND_SELF_STR, expand_one,
+        expand_to_command_and_args,
+    },
+    function,
+    highlight::file_tester::FileTester,
+    history::all_paths_are_valid,
+    operation_context::OperationContext,
+    parse_constants::{
+        ParseKeyword, ParseTokenType, ParseTreeFlags, SourceRange, StatementDecoration,
+    },
+    parse_util::{
+        MaybeParentheses, get_process_first_token_offset, locate_cmdsubst_range, slice_length,
+    },
+    path::{path_as_implicit_cd, path_get_cdpath, path_get_path, paths_are_same_file},
+    terminal::Outputter,
+    text_face::{ResettableStyle, SpecifiedTextFace, TextFace, UnderlineStyle, parse_text_face},
+    threads::assert_is_background_thread,
+    tokenizer::{PipeOrRedir, variable_assignment_equals_pos},
 };
-use crate::builtins::shared::builtin_exists;
-use crate::common::{valid_var_name, valid_var_name_char};
-use crate::complete::complete_wrap_map;
-use crate::env::{EnvVar, Environment};
-use crate::expand::{
-    ExpandFlags, ExpandResultCode, PROCESS_EXPAND_SELF_STR, expand_one, expand_to_command_and_args,
-};
-use crate::function;
-use crate::highlight::file_tester::FileTester;
-use crate::history::all_paths_are_valid;
-use crate::operation_context::OperationContext;
-use crate::parse_constants::{
-    ParseKeyword, ParseTokenType, ParseTreeFlags, SourceRange, StatementDecoration,
-};
-use crate::parse_util::{
-    MaybeParentheses, get_process_first_token_offset, locate_cmdsubst_range, slice_length,
-};
-use crate::path::{path_as_implicit_cd, path_get_cdpath, path_get_path, paths_are_same_file};
-use crate::terminal::Outputter;
-use crate::text_face::{
-    ResettableStyle, SpecifiedTextFace, TextFace, UnderlineStyle, parse_text_face,
-};
-use crate::threads::assert_is_background_thread;
-use crate::tokenizer::{PipeOrRedir, variable_assignment_equals_pos};
 use fish_color::Color;
-use fish_common::{ASCII_MAX, EXPAND_RESERVED_BASE, EXPAND_RESERVED_END};
 use fish_feature_flags::{FeatureFlag, feature_test};
 use fish_wcstringutil::string_prefixes_string;
-use fish_widestring::{L, WExt as _, WString, wstr};
-use std::collections::HashMap;
-use std::collections::hash_map::Entry;
+use fish_widestring::{
+    ASCII_MAX, EXPAND_RESERVED_BASE, EXPAND_RESERVED_END, L, WExt as _, WString, wstr,
+};
+use std::collections::{HashMap, hash_map::Entry};
 
 use super::file_tester::IsFile;
 

src/history/history.rs

@@ -36,13 +36,13 @@
     path::{path_get_config, path_get_data, path_is_valid},
     prelude::*,
     threads::{ThreadPool, assert_is_background_thread},
-    wildcard::{ANY_STRING, wildcard_match},
+    wildcard::wildcard_match,
     wutil::{FileId, INVALID_FILE_ID, file_id_for_file, wrealpath, wstat, wunlink},
 };
 use bitflags::bitflags;
 use fish_common::UnescapeStringStyle;
 use fish_wcstringutil::{subsequence_in_string, trim};
-use fish_widestring::subslice_position;
+use fish_widestring::{ANY_STRING, subslice_position};
 use lru::LruCache;
 use nix::{fcntl::OFlag, sys::stat::Mode};
 use rand::Rng as _;

src/input_common.rs

@@ -14,9 +14,9 @@
 };
 use crate::universal_notifier::default_notifier;
 use crate::wutil::{fish_is_pua, fish_wcstol};
-use fish_common::{fish_reserved_codepoint, read_blocked};
+use fish_common::read_blocked;
 use fish_feature_flags::{FeatureFlag, feature_test};
-use fish_widestring::encode_byte_to_char;
+use fish_widestring::{encode_byte_to_char, fish_reserved_codepoint};
 use nix::sys::{select::FdSet, signal::SigSet, time::TimeSpec};
 use std::cell::{RefCell, RefMut};
 use std::collections::VecDeque;

src/parse_util.rs

@@ -1,33 +1,35 @@
 //! Various mostly unrelated utility functions related to parsing, loading and evaluating fish code.
-use crate::ast::{
-    self, Ast, Keyword as _, Kind, Leaf as _, Node, NodeVisitor, Token as _, Traversal,
-    is_same_node,
+use crate::{
+    ast::{
+        self, Ast, Keyword as _, Kind, Leaf as _, Node, NodeVisitor, Token as _, Traversal,
+        is_same_node,
+    },
+    builtins::shared::builtin_exists,
+    common::{unescape_string, valid_var_name, valid_var_name_char},
+    expand::{
+        BRACE_BEGIN, BRACE_END, BRACE_SEP, ExpandFlags, ExpandResultCode, INTERNAL_SEPARATOR,
+        VARIABLE_EXPAND, VARIABLE_EXPAND_EMPTY, VARIABLE_EXPAND_SINGLE, expand_one,
+        expand_to_command_and_args,
+    },
+    operation_context::OperationContext,
+    parse_constants::{
+        ERROR_BAD_VAR_CHAR1, ERROR_BRACKETED_VARIABLE_QUOTED1, ERROR_BRACKETED_VARIABLE1,
+        ERROR_NO_VAR_NAME, ERROR_NOT_ARGV_AT, ERROR_NOT_ARGV_COUNT, ERROR_NOT_ARGV_STAR,
+        ERROR_NOT_PID, ERROR_NOT_STATUS, INVALID_BREAK_ERR_MSG, INVALID_CONTINUE_ERR_MSG,
+        INVALID_PIPELINE_CMD_ERR_MSG, ParseError, ParseErrorCode, ParseErrorList, ParseIssue,
+        ParseKeyword, ParseTokenType, ParseTreeFlags, PipelinePosition, SourceRange,
+        StatementDecoration, UNKNOWN_BUILTIN_ERR_MSG, parse_error_offset_source_start,
+    },
+    prelude::*,
+    tokenizer::{
+        TOK_ACCEPT_UNFINISHED, TOK_SHOW_COMMENTS, Tok, TokenType, Tokenizer, comment_end,
+        is_token_delimiter, quote_end,
+    },
 };
-use crate::builtins::shared::builtin_exists;
-use crate::common::{unescape_string, valid_var_name, valid_var_name_char};
-use crate::expand::{
-    BRACE_BEGIN, BRACE_END, BRACE_SEP, ExpandFlags, ExpandResultCode, INTERNAL_SEPARATOR,
-    VARIABLE_EXPAND, VARIABLE_EXPAND_EMPTY, VARIABLE_EXPAND_SINGLE, expand_one,
-    expand_to_command_and_args,
-};
-use crate::operation_context::OperationContext;
-use crate::parse_constants::{
-    ERROR_BAD_VAR_CHAR1, ERROR_BRACKETED_VARIABLE_QUOTED1, ERROR_BRACKETED_VARIABLE1,
-    ERROR_NO_VAR_NAME, ERROR_NOT_ARGV_AT, ERROR_NOT_ARGV_COUNT, ERROR_NOT_ARGV_STAR, ERROR_NOT_PID,
-    ERROR_NOT_STATUS, INVALID_BREAK_ERR_MSG, INVALID_CONTINUE_ERR_MSG,
-    INVALID_PIPELINE_CMD_ERR_MSG, ParseError, ParseErrorCode, ParseErrorList, ParseIssue,
-    ParseKeyword, ParseTokenType, ParseTreeFlags, PipelinePosition, SourceRange,
-    StatementDecoration, UNKNOWN_BUILTIN_ERR_MSG, parse_error_offset_source_start,
-};
-use crate::prelude::*;
-use crate::tokenizer::{
-    TOK_ACCEPT_UNFINISHED, TOK_SHOW_COMMENTS, Tok, TokenType, Tokenizer, comment_end,
-    is_token_delimiter, quote_end,
-};
-use crate::wildcard::{ANY_CHAR, ANY_STRING, ANY_STRING_RECURSIVE};
 use fish_common::{UnescapeFlags, UnescapeStringStyle, help_section};
 use fish_feature_flags::{FeatureFlag, feature_test};
 use fish_wcstringutil::{count_newlines, truncate};
+use fish_widestring::{ANY_CHAR, ANY_STRING, ANY_STRING_RECURSIVE};
 use std::ops::Range;
 use std::{iter, ops};
 

src/reader/reader.rs

@@ -17,9 +17,11 @@
 //! control-C from generating SIGINT, so failing to disable these would prevent cancellation of wildcard
 //! expansion, etc.
 
-use super::history_search::{ReaderHistorySearch, SearchMode, smartcase_flags};
-use super::iothreads::{self, Debouncers};
-use super::word_motion::{MoveWordDir, MoveWordStateMachine, MoveWordStyle};
+use super::{
+    history_search::{ReaderHistorySearch, SearchMode, smartcase_flags},
+    iothreads::{self, Debouncers},
+    word_motion::{MoveWordDir, MoveWordStateMachine, MoveWordStyle},
+};
 use crate::{
     abbrs::{self, abbrs_match},
     ast::{self, Kind, is_same_node},
@@ -112,12 +114,11 @@
 use assert_matches::assert_matches;
 use errno::{Errno, errno};
 use fish_common::{
-    EscapeFlags, EscapeStringStyle, ScopeGuard, ScopeGuarding, UTF8_BOM_WCHAR,
-    exit_without_destructors, get_obfuscation_read_char, help_section,
-    restore_term_foreground_process_group_for_exit, write_loop,
+    EscapeFlags, EscapeStringStyle, ScopeGuard, ScopeGuarding, exit_without_destructors,
+    get_obfuscation_read_char, help_section, restore_term_foreground_process_group_for_exit,
+    write_loop,
 };
-use fish_fallback::fish_wcwidth;
-use fish_fallback::lowercase;
+use fish_fallback::{fish_wcwidth, lowercase};
 use fish_feature_flags::FeatureFlag;
 use fish_util::{perror, write_to_fd};
 use fish_wcstringutil::{
@@ -125,12 +126,11 @@
     join_strings, string_prefixes_string, string_prefixes_string_case_insensitive,
     string_prefixes_string_maybe_case_insensitive,
 };
-use fish_widestring::ELLIPSIS_CHAR;
+use fish_widestring::{ELLIPSIS_CHAR, UTF8_BOM_WCHAR};
 use libc::{
     _POSIX_VDISABLE, EIO, EISDIR, ENOTTY, ESRCH, O_NONBLOCK, O_RDONLY, SIGINT, STDERR_FILENO,
     STDIN_FILENO, STDOUT_FILENO, VMIN, VQUIT, VSUSP, VTIME, c_char,
 };
-use nix::unistd::setpgid;
 use nix::{
     fcntl::OFlag,
     sys::{
@@ -138,7 +138,7 @@
         stat::Mode,
         termios::{self, SetArg, Termios, tcgetattr, tcsetattr},
     },
-    unistd::{getpgrp, getpid},
+    unistd::{getpgrp, getpid, setpgid},
 };
 use std::{
     borrow::Cow,

src/wildcard.rs

@@ -1,19 +1,15 @@
 // Enumeration of all wildcard types.
 
-use fish_common::WILDCARD_RESERVED_BASE;
-use fish_widestring::char_offset;
-use nix::unistd::AccessFlags;
-use std::cell::LazyCell;
-use std::cmp::Ordering;
-use std::collections::HashSet;
-use std::os::unix::fs::MetadataExt as _;
-
-use crate::common::{WSL, is_windows_subsystem_for_linux, unescape_string};
-use crate::complete::{CompleteFlags, Completion, CompletionReceiver, PROG_COMPLETE_SEP};
-use crate::expand::ExpandFlags;
-use crate::prelude::*;
-use crate::wutil::dir_iter::DirEntryType;
-use crate::wutil::{dir_iter::DirEntry, lwstat, waccess};
+use crate::{
+    common::{WSL, is_windows_subsystem_for_linux, unescape_string},
+    complete::{CompleteFlags, Completion, CompletionReceiver, PROG_COMPLETE_SEP},
+    expand::ExpandFlags,
+    prelude::*,
+    wutil::{
+        dir_iter::{DirEntry, DirEntryType},
+        lwstat, waccess,
+    },
+};
 use fish_common::{UnescapeFlags, UnescapeStringStyle};
 use fish_fallback::wcscasecmp;
 use fish_feature_flags::{FeatureFlag, feature_test};
@@ -21,6 +17,9 @@
     CaseSensitivity, string_fuzzy_match_string, string_suffixes_string_case_insensitive,
     strip_executable_suffix,
 };
+use fish_widestring::{ANY_CHAR, ANY_STRING, ANY_STRING_RECURSIVE};
+use nix::unistd::AccessFlags;
+use std::{cell::LazyCell, cmp::Ordering, collections::HashSet, os::unix::fs::MetadataExt as _};
 
 localizable_consts!(
     COMPLETE_EXEC_DESC "command"
@@ -31,17 +30,6 @@
     COMPLETE_DIRECTORY_DESC "directory"
 );
 
-/// Character representing any character except '/' (slash).
-pub const ANY_CHAR: char = char_offset(WILDCARD_RESERVED_BASE, 0);
-/// Character representing any character string not containing '/' (slash).
-pub const ANY_STRING: char = char_offset(WILDCARD_RESERVED_BASE, 1);
-/// Character representing any character string.
-pub const ANY_STRING_RECURSIVE: char = char_offset(WILDCARD_RESERVED_BASE, 2);
-/// This is a special pseudo-char that is not used other than to mark the
-/// end of the special characters so we can sanity check the enum range.
-#[allow(dead_code)]
-pub const ANY_SENTINEL: char = char_offset(WILDCARD_RESERVED_BASE, 3);
-
 #[derive(PartialEq)]
 pub enum WildcardResult {
     /// The wildcard did not match.

src/wutil/mod.rs

@@ -15,11 +15,11 @@
     topic_monitor::Topic,
 };
 use errno::{Errno, set_errno};
-use fish_common::fish_reserved_codepoint;
 use fish_util::{perror, write_to_fd};
 use fish_wcstringutil::join_strings;
 use fish_widestring::{
-    IntoCharIter, L, WExt as _, WString, str2bytes_callback, wcs2osstring, wcs2zstring, wstr,
+    IntoCharIter, L, WExt as _, WString, fish_reserved_codepoint, str2bytes_callback, wcs2osstring,
+    wcs2zstring, wstr,
 };
 use nix::unistd::AccessFlags;
 use std::{