//! String expansion functions. These functions perform several kinds of parameter expansion. There
//! are a lot of issues with regards to memory allocation. Overall, these functions would benefit
//! from using a more clever memory allocation scheme, perhaps an evil combination of talloc,
//! string buffers and reference counting.

use crate::{
    builtins::{
        STATUS_CMD_ERROR, STATUS_CMD_UNKNOWN, STATUS_EXPAND_ERROR, STATUS_ILLEGAL_CMD,
        STATUS_INVALID_ARGS, STATUS_NOT_EXECUTABLE, STATUS_READ_TOO_MUCH,
        STATUS_UNMATCHED_WILDCARD,
    },
    common::valid_var_name_char,
    complete::{CompleteFlags, Completion, CompletionList, CompletionReceiver},
    env::{EnvVar, Environment},
    exec::exec_subshell_for_expand,
    history::{History, history_id},
    operation_context::OperationContext,
    parse_constants::{ParseError, ParseErrorCode, ParseErrorList, SOURCE_LOCATION_UNKNOWN},
    parse_util::{expand_variable_error, locate_cmdsubst_range},
    path::path_apply_working_directory,
    prelude::*,
    wildcard::{WildcardResult, wildcard_expand_string, wildcard_has_internal},
    wutil::{normalize_path, wcstoi, wcstoi_partial},
};
use bitflags::bitflags;
use fish_common::{
    EscapeFlags, EscapeStringStyle, UnescapeFlags, UnescapeStringStyle, escape, escape_string,
    escape_string_for_double_quotes, unescape_string,
};
use fish_feature_flags::{FeatureFlag, feature_test};
use fish_util::wcsfilecmp_glob;
use fish_wcstringutil::{join_strings, trim};
use fish_widestring::{
    ANY_CHAR, ANY_STRING, ANY_STRING_RECURSIVE, BRACE_BEGIN, BRACE_END, BRACE_SEP, BRACE_SPACE,
    HOME_DIRECTORY, INTERNAL_SEPARATOR, PROCESS_EXPAND_SELF, VARIABLE_EXPAND,
    VARIABLE_EXPAND_EMPTY, VARIABLE_EXPAND_SINGLE, osstr2wcstring,
};
use nix::unistd::{User, getpid};

bitflags! {
    /// Set of flags controlling expansions.
    #[derive(Copy, Clone, Default)]
    pub struct ExpandFlags : u16 {
        /// Fail expansion if there is a command substitution.
        const FAIL_ON_CMDSUBST = 1 << 0;
        /// Skip variable expansion.
        const SKIP_VARIABLES = 1 << 1;
        /// Skip wildcard expansion.
        const SKIP_WILDCARDS = 1 << 2;
        /// The expansion is being done for tab or auto completions. Returned completions may have the
        /// wildcard as a prefix instead of a match.
        const FOR_COMPLETIONS = 1 << 3;
        /// Only match files that are executable by the current user.
        const EXECUTABLES_ONLY = 1 << 4;
        /// Only match directories.
        const DIRECTORIES_ONLY = 1 << 5;
        /// Generate descriptions, stored in the description field of completions.
        const GEN_DESCRIPTIONS = 1 << 6;
        /// Un-expand home directories to tildes after.
        const PRESERVE_HOME_TILDES = 1 << 7;
        /// Allow fuzzy matching.
        const FUZZY_MATCH = 1 << 8;
        /// Allows matching a leading dot even if the wildcard does not contain one.
        /// By default, wildcards only match a leading dot literally; this is why e.g. '*' does not
        /// match hidden files.
        const ALLOW_NONLITERAL_LEADING_DOT = 1 << 10;
        /// Do expansions specifically to support cd. This means using CDPATH as a list of potential
        /// working directories, and to use logical instead of physical paths.
        const SPECIAL_FOR_CD = 1 << 11;
        /// Do expansions specifically for cd autosuggestion. This is to differentiate between cd
        /// completions and cd autosuggestions.
        const SPECIAL_FOR_CD_AUTOSUGGESTION = 1 << 12;
        /// Do expansions specifically to support external command completions. This means using PATH as
        /// a list of potential working directories.
        const SPECIAL_FOR_COMMAND = 1 << 13;
        /// The token has an unclosed brace, so don't add a space.
        const NO_SPACE_FOR_UNCLOSED_BRACE = 1 << 14;
        /// Skip command substitutions.
        const SKIP_CMDSUBST = 1 << 15;
    }
}

impl ExpandResult {
    pub fn new(result: ExpandResultCode) -> Self {
        Self { result, status: 0 }
    }
    pub fn ok() -> Self {
        Self::new(ExpandResultCode::Ok)
    }
    /// Make an error value with the given status.
    pub fn make_error(status: libc::c_int) -> Self {
        assert_ne!(status, 0, "status cannot be 0 for an error result");
        Self {
            result: ExpandResultCode::Error,
            status,
        }
    }
}

impl PartialEq<ExpandResultCode> for ExpandResult {
    fn eq(&self, other: &ExpandResultCode) -> bool {
        self.result == *other
    }
}

/// Perform various forms of expansion on in, such as tilde expansion (\~USER becomes the users home
/// directory), variable expansion (\$VAR_NAME becomes the value of the environment variable
/// VAR_NAME), cmdsubst expansion and wildcard expansion. The results are inserted into the list
/// out.
///
/// If the parameter does not need expansion, it is copied into the list out.
///
/// \param input The parameter to expand
/// \param output The list to which the result will be appended.
/// \param ctx The parser, variables, and cancellation checker for this operation.  The parser may
/// be null. \param errors Resulting errors, or nullptr to ignore
///
/// Return An expand_result_t.
/// wildcard_no_match and wildcard_match are normal exit conditions used only on
/// strings containing wildcards to tell if the wildcard produced any matches.
pub fn expand_string(
    input: WString,
    out_completions: &mut CompletionList,
    flags: ExpandFlags,
    ctx: &mut OperationContext,
    errors: Option<&mut ParseErrorList>,
) -> ExpandResult {
    let completions = std::mem::take(out_completions);
    let mut recv = CompletionReceiver::from_list(completions, ctx.expansion_limit);
    let result = expand_to_receiver(input, &mut recv, flags, ctx, errors);
    *out_completions = recv.take();
    result
}

/// Variant of string that inserts its results into a completion_receiver_t.
pub fn expand_to_receiver(
    input: WString,
    out_completions: &mut CompletionReceiver,
    flags: ExpandFlags,
    ctx: &mut OperationContext,
    errors: Option<&mut ParseErrorList>,
) -> ExpandResult {
    Expander::expand_string(input, out_completions, flags, ctx, errors)
}

/// expand_one is identical to expand_string, except it will fail if in expands to more than one
/// string. This is used for expanding command names.
///
/// \param inout_str The parameter to expand in-place
/// \param ctx The parser, variables, and cancellation checker for this operation. The parser may be
/// null.
/// \param errors Resulting errors, or nullptr to ignore
///
/// Return Whether expansion succeeded.
pub fn expand_one(
    s: &mut WString,
    flags: ExpandFlags,
    ctx: &mut OperationContext,
    errors: Option<&mut ParseErrorList>,
) -> bool {
    if !flags.contains(ExpandFlags::FOR_COMPLETIONS) && expand_is_clean(s) {
        return true;
    }

    let mut completions = CompletionList::new();
    let input = std::mem::take(s);

    let ok = expand_string(input, &mut completions, flags, ctx, errors) == ExpandResultCode::Ok;

    if ok && completions.len() == 1 {
        *s = std::mem::take(&mut completions[0].completion);
        return true;
    }

    false
}

/// Expand a command string like $HOME/bin/cmd into a command and list of arguments.
/// Return the command and arguments by reference.
/// If the expansion resulted in no or an empty command, the command will be an empty string. Note
/// that API does not distinguish between expansion resulting in an empty command (''), and
/// expansion resulting in no command (e.g. unset variable).
/// If `skip_wildcards` is true, then do not do wildcard expansion
/// Return an expand error.
pub fn expand_to_command_and_args(
    instr: &wstr,
    ctx: &mut OperationContext<'_>,
    out_cmd: &mut WString,
    mut out_args: Option<&mut Vec<WString>>,
    errors: Option<&mut ParseErrorList>,
    skip_wildcards: bool,
) -> ExpandResult {
    // Fast path.
    if expand_is_clean(instr) {
        *out_cmd = instr.to_owned();
        return ExpandResult::ok();
    }

    let mut eflags = ExpandFlags::FAIL_ON_CMDSUBST;
    if skip_wildcards {
        eflags |= ExpandFlags::SKIP_WILDCARDS;
    }

    let mut completions = CompletionList::new();
    let expand_err = expand_string(instr.to_owned(), &mut completions, eflags, ctx, errors);
    if expand_err == ExpandResultCode::Ok {
        // The first completion is the command, any remaining are arguments.
        let mut completions = completions.into_iter();
        if let Some(comp) = completions.next() {
            *out_cmd = comp.completion;
        }
        if let Some(ref mut out_args) = out_args {
            for comp in completions {
                out_args.push(comp.completion);
            }
        }
    }

    expand_err
}

/// Convert the variable value to a human readable form, i.e. escape things, handle arrays, etc.
/// Suitable for pretty-printing.
pub fn expand_escape_variable(var: &EnvVar) -> WString {
    let mut buff = WString::new();

    let lst = var.as_list();
    for el in lst {
        if !buff.is_empty() {
            buff.push_str(" ");
        }

        // We want to use quotes if we have more than one string, or the string contains a space.
        let prefer_quotes = lst.len() > 1 || el.contains(' ');
        if prefer_quotes && is_quotable(el) {
            buff.push('\'');
            buff.push_utfstr(el);
            buff.push('\'');
        } else {
            buff.push_utfstr(&escape(el));
        }
    }
    buff
}

/// Convert a string value to a human readable form, i.e. escape things, handle arrays, etc.
/// Suitable for pretty-printing.
pub fn expand_escape_string(el: &wstr) -> WString {
    let mut buff = WString::new();
    let prefer_quotes = el.contains(' ');
    if prefer_quotes && is_quotable(el) {
        buff.push('\'');
        buff.push_utfstr(el);
        buff.push('\'');
    } else {
        buff.push_utfstr(&escape(el));
    }
    buff
}

/// Perform tilde expansion and nothing else on the specified string, which is modified in place.
///
/// \param input the string to tilde expand
pub fn expand_tilde(input: &mut WString, vars: &dyn Environment) {
    if input.chars().next() == Some('~') {
        input.replace_range(0..1, wstr::from_char_slice(&[HOME_DIRECTORY]));
        expand_home_directory(input, vars);
    }
}

/// Perform the opposite of tilde expansion on the string.
pub fn replace_home_directory_with_tilde(s: impl Into<WString>, vars: &dyn Environment) -> WString {
    let mut result = s.into();
    // Only absolute paths get this treatment.
    if result.starts_with(L!("/")) {
        let mut home_directory = L!("~").to_owned();
        expand_tilde(&mut home_directory, vars);
        // If we can't get a home directory, don't replace anything.
        // This is the case e.g. with --no-execute
        if home_directory.is_empty() {
            return result;
        }
        if !home_directory.ends_with(L!("/")) {
            home_directory.push('/');
        }

        // Now check if the home_directory prefixes the string.
        if result.starts_with(&home_directory) {
            // Success
            result.replace_range(0..home_directory.len(), L!("~/"));
        }
    }
    result
}

/// Characters which make a string unclean if they are the first character of the string. See \c
/// expand_is_clean().
const UNCLEAN_FIRST: &wstr = L!("~%");
/// Unclean characters. See \c expand_is_clean().
const UNCLEAN: &wstr = L!("$*?\\\"'({})");

/// Test if the specified argument is clean, i.e. it does not contain any tokens which need to be
/// expanded or otherwise altered. Clean strings can be passed through expand_string and expand_one
/// without changing them. About two thirds of all strings are clean, so skipping expansion on them
/// actually does save a small amount of time, since it avoids multiple memory allocations during
/// the expansion process.
///
/// \param in the string to test
fn expand_is_clean(input: &wstr) -> bool {
    if input.is_empty() {
        return true;
    }

    // Test characters that have a special meaning in the first character position.
    if UNCLEAN_FIRST.contains(input.as_char_slice()[0]) {
        return false;
    }

    // Test characters that have a special meaning in any character position.
    !input.chars().any(|c| UNCLEAN.contains(c))
}

/// Append a syntax error to the given error list.
macro_rules! append_syntax_error {
    (
        $errors:expr, $source_start:expr,
        $fmt:expr $(, $arg:expr )* $(,)?
    ) => {
        if let Some(ref mut errors) = $errors.as_mut() {
            let mut error = ParseError::default();
            error.source_start = $source_start;
            error.source_length = 0;
            error.code = ParseErrorCode::Syntax;
            error.text = wgettext_fmt!($fmt $(, $arg)*);
            errors.push(error);
        }
    }
}

/// Append a cmdsub error to the given error list. But only do so if the error hasn't already been
/// recorded. This is needed because command substitution is a recursive process and some errors
/// could consequently be recorded more than once.
macro_rules! append_cmdsub_error {
    (
        $errors:expr, $source_start:expr, $source_end:expr,
        $text:expr $(,)?
    ) => {
        if let Some(ref mut errors) = $errors.as_mut() {
            let mut error = ParseError::default();
            error.source_start = $source_start;
            error.source_length = $source_end - $source_start + 1;
            error.code = ParseErrorCode::CmdSubst;
            error.text = $text.to_owned();
            if !errors.iter().any(|e| e.text == error.text) {
                errors.push(error);
            }
        }
    };
}

/// Append an overflow error, when expansion produces too much data.
fn append_overflow_error(
    errors: &mut Option<&mut ParseErrorList>,
    source_start: Option<usize>,
) -> ExpandResult {
    if let Some(errors) = errors {
        errors.push(ParseError {
            source_start: source_start.unwrap_or(SOURCE_LOCATION_UNKNOWN),
            source_length: 0,
            code: ParseErrorCode::Generic,
            text: wgettext!("Expansion produced too many results").to_owned(),
        });
    }
    ExpandResult::make_error(STATUS_EXPAND_ERROR)
}

/// Test if the specified string does not contain character which can not be used inside a quoted
/// string.
fn is_quotable(s: &wstr) -> bool {
    !s.chars().any(|c| "\n\t\r\x08\x1B".contains(c))
}

enum ParseSliceError {
    ZeroIndex,
    InvalidIndex,
}

/// Parse an array slicing specification Returns 0 on success. If a parse error occurs, returns the
/// index of the bad token. Note that 0 can never be a bad index because the string always starts
/// with [.
fn parse_slice(
    input: &wstr,
    idx: &mut Vec<i64>,
    array_size: usize,
) -> Result<usize, (usize, ParseSliceError)> {
    let size = i64::try_from(array_size).unwrap();
    let mut pos = 1; // skip past the opening square bracket

    loop {
        while input.char_at(pos).is_whitespace() || input.char_at(pos) == INTERNAL_SEPARATOR {
            pos += 1;
        }
        if input.char_at(pos) == ']' {
            pos += 1;
            break;
        }

        let tmp = if idx.is_empty() && input.char_at(pos) == '.' && input.char_at(pos + 1) == '.' {
            // If we are at the first index expression, a missing start-index means the range starts
            // at the first item.
            1 // first index
        } else {
            let mut consumed = 0;
            match wcstoi_partial(&input[pos..], wcstoi::Options::default(), &mut consumed) {
                Ok(tmp) => {
                    if tmp == 0 {
                        // Explicitly refuse $foo[0] as valid syntax, regardless of whether or
                        // not we're going to show an error if the index ultimately evaluates
                        // to zero. This will help newcomers to fish avoid a common off-by-one
                        // error. See #4862.
                        return Err((pos, ParseSliceError::ZeroIndex));
                    }
                    pos += consumed;
                    // Skip trailing whitespace.
                    pos += input[pos..]
                        .chars()
                        .take_while(|c| c.is_whitespace())
                        .count();
                    tmp
                }
                Err(_error) => {
                    // We don't test `*end` as is typically done because we expect it to not
                    // be the null char. Ignore the case of errno==-1 because it means the end
                    // char wasn't the null char.
                    return Err((pos, ParseSliceError::InvalidIndex));
                }
            }
        };

        let mut i1 = if tmp > -1 { tmp } else { size + tmp + 1 };
        while input.char_at(pos) == INTERNAL_SEPARATOR {
            pos += 1;
        }
        if input.char_at(pos) == '.' && input.char_at(pos + 1) == '.' {
            pos += 2;
            while input.char_at(pos) == INTERNAL_SEPARATOR {
                pos += 1;
            }
            while input.char_at(pos).is_whitespace() {
                pos += 1; // Allow the space in "[.. ]".
            }

            // If we are at the last index range expression  then a missing end-index means the
            // range spans until the last item.
            let tmp1 = if input.char_at(pos) == ']' {
                -1 // last index
            } else {
                let mut consumed = 0;
                match wcstoi_partial(&input[pos..], wcstoi::Options::default(), &mut consumed) {
                    Ok(tmp) => {
                        if tmp == 0 {
                            return Err((pos, ParseSliceError::ZeroIndex));
                        }
                        pos += consumed;
                        // Skip trailing whitespace.
                        pos += input[pos..]
                            .chars()
                            .take_while(|c| c.is_whitespace())
                            .count();
                        tmp
                    }
                    Err(_error) => {
                        return Err((pos, ParseSliceError::InvalidIndex));
                    }
                }
            };

            let mut i2 = if tmp1 > -1 { tmp1 } else { size + tmp1 + 1 };
            // Skip sequences that are entirely outside.
            // This means "17..18" expands to nothing if there are less than 17 elements.
            if i1 > size && i2 > size {
                continue;
            }
            let mut direction = if i2 < i1 { -1 } else { 1 };
            // If only the beginning is negative, always go reverse.
            // If only the end, always go forward.
            // Prevents `[x..-1]` from going reverse if less than x elements are there.
            if (tmp1 > -1) != (tmp > -1) {
                direction = if tmp1 > -1 { -1 } else { 1 };
            } else {
                // Clamp to array size when not forcing direction
                // - otherwise "2..-1" clamps both to 1 and then becomes "1..1".
                i1 = i1.min(size);
                i2 = i2.min(size);
            }
            let mut jjj = i1;
            while jjj * direction <= i2 * direction {
                idx.push(jjj);
                jjj += direction;
            }
            continue;
        }

        idx.push(i1);
    }

    Ok(pos)
}

/// Expand all environment variables in the string *ptr.
///
/// This function is slow, fragile and complicated. There are lots of little corner cases, like
/// $$foo should do a double expansion, $foo$bar should not double expand bar, etc.
///
/// This function operates on strings backwards, starting at last_idx.
///
/// Note: last_idx is considered to be where it previously finished processing. This means it
/// actually starts operating on last_idx-1. As such, to process a string fully, pass string.size()
/// as last_idx instead of string.size()-1.
///
/// Return the result of expansion.
fn expand_variables(
    instr: WString,
    out: &mut CompletionReceiver,
    last_idx: usize,
    vars: &dyn Environment,
    errors: &mut Option<&mut ParseErrorList>,
) -> ExpandResult {
    // last_idx may be 1 past the end of the string, but no further.
    assert!(last_idx <= instr.len(), "Invalid last_idx");
    if last_idx == 0 {
        if !out.add(instr) {
            return append_overflow_error(errors, None);
        }
        return ExpandResult::ok();
    }

    // Locate the last VARIABLE_EXPAND or VARIABLE_EXPAND_SINGLE
    let mut is_single = false;
    let mut varexp_char_idx = last_idx;
    loop {
        let done = varexp_char_idx == 0;
        varexp_char_idx = varexp_char_idx.wrapping_sub(1);
        if done {
            break;
        }
        let c = instr.as_char_slice()[varexp_char_idx];
        if [VARIABLE_EXPAND, VARIABLE_EXPAND_SINGLE].contains(&c) {
            is_single = c == VARIABLE_EXPAND_SINGLE;
            break;
        }
    }
    if varexp_char_idx == usize::MAX {
        // No variable expand char, we're done.
        if !out.add(instr) {
            return append_overflow_error(errors, None);
        }
        return ExpandResult::ok();
    }

    // Get the variable name.
    let var_name_start = varexp_char_idx + 1;
    let mut var_name_stop = var_name_start;
    while var_name_stop < instr.len() {
        let nc = instr.as_char_slice()[var_name_stop];
        if nc == VARIABLE_EXPAND_EMPTY {
            var_name_stop += 1;
            break;
        }
        if !valid_var_name_char(nc) {
            break;
        }
        var_name_stop += 1;
    }
    assert!(
        var_name_stop >= var_name_start,
        "Bogus variable name indexes"
    );

    // Get the variable name as a string, then try to get the variable from env.
    let var_name = &instr[var_name_start..var_name_stop];

    // It's an error if the name is empty.
    if var_name.is_empty() {
        if let Some(errors) = errors {
            expand_variable_error(
                &instr,
                0, /* global_token_pos */
                varexp_char_idx,
                errors,
            );
        }
        return ExpandResult::make_error(STATUS_EXPAND_ERROR);
    }

    // Do a dirty hack to make sliced history fast (#4650). We expand from either a variable, or a
    // history_t. Note that "history" is read only in env.rs so it's safe to special-case it in
    // this way (it cannot be shadowed, etc).
    let mut history = None;
    let mut var = None;
    if var_name == "history" {
        history = Some(History::new(history_id(vars)));
    } else if var_name.as_char_slice() != [VARIABLE_EXPAND_EMPTY] {
        var = vars.get(var_name);
    }

    // Parse out any following slice.
    // Record the end of the variable name and any following slice.
    let mut var_name_and_slice_stop = var_name_stop;
    let mut all_values = true;
    let slice_start = var_name_stop;
    let mut var_idx_list = vec![];

    if instr.as_char_slice().get(slice_start) == Some(&'[') {
        all_values = false;
        // If a variable is missing, behave as though we have one value, so that $var[1] always
        // works.
        let mut effective_val_count = 1;
        if let Some(ref var) = var {
            effective_val_count = var.as_list().len();
        } else if let Some(ref history) = history {
            effective_val_count = history.size();
        }
        match parse_slice(
            &instr[slice_start..],
            &mut var_idx_list,
            effective_val_count,
        ) {
            Ok(offset) => {
                var_name_and_slice_stop = slice_start + offset;
            }
            Err((bad_pos, error)) => {
                match error {
                    ParseSliceError::ZeroIndex => {
                        append_syntax_error!(
                            errors,
                            slice_start + bad_pos,
                            "array indices start at 1, not 0."
                        );
                    }
                    ParseSliceError::InvalidIndex => {
                        append_syntax_error!(errors, slice_start + bad_pos, "Invalid index value");
                    }
                }
                return ExpandResult::make_error(STATUS_EXPAND_ERROR);
            }
        }
    }
    let var_idx_list = var_idx_list.iter().filter_map(|&n| n.try_into().ok());

    if var.is_none() && history.is_none() {
        // Expanding a non-existent variable.
        if !is_single {
            // Normal expansions of missing variables successfully expand to nothing.
            return ExpandResult::ok();
        } else {
            // Expansion to single argument.
            // Replace the variable name and slice with VARIABLE_EXPAND_EMPTY.
            let mut res = instr[..varexp_char_idx].to_owned();
            if res.as_char_slice().last() == Some(&VARIABLE_EXPAND_SINGLE) {
                res.push(VARIABLE_EXPAND_EMPTY);
            }
            res.push_utfstr(&instr[var_name_and_slice_stop..]);
            return expand_variables(res, out, varexp_char_idx, vars, errors);
        }
    }

    // Ok, we have a variable or a history. Let's expand it.
    // Start by respecting the sliced elements.
    assert!(
        var.is_some() || history.is_some(),
        "Should have variable or history here",
    );
    let mut var_item_list = vec![];
    if all_values {
        var_item_list = if let Some(ref history) = history {
            history.get_history()
        } else {
            var.as_ref().unwrap().as_list().to_vec()
        };
    } else {
        // We have to respect the slice.
        if let Some(ref history) = history {
            // Ask history to map indexes to item strings.
            // Note this may have missing entries for out-of-bounds.
            let item_map = history.items_at_indexes(var_idx_list.clone());
            for item_index in var_idx_list {
                if let Some(item) = item_map.get(&item_index) {
                    var_item_list.push(item.clone());
                }
            }
        } else {
            let all_var_items = var.as_ref().unwrap().as_list();
            for item_index in var_idx_list {
                // Check that we are within array bounds. If not, skip the element. Note:
                // Negative indices (`echo $foo[-1]`) are already converted to positive ones
                // here, So tmp < 1 means it's definitely not in.
                // Note we are 1-based.
                if item_index >= 1 && item_index <= all_var_items.len() {
                    var_item_list.push(all_var_items[item_index - 1].clone());
                }
            }
        }
    }

    if is_single {
        // Quoted expansion. Here we expect the variable's delimiter.
        // Note history always has a space delimiter.
        let delimit = if history.is_some() {
            ' '
        } else {
            var.as_ref().unwrap().delimiter()
        };
        let mut res = instr[..varexp_char_idx].to_owned();
        if !res.is_empty() {
            if res.as_char_slice().last() != Some(&VARIABLE_EXPAND_SINGLE) {
                res.push(INTERNAL_SEPARATOR);
            } else if var_item_list.is_empty() || var_item_list[0].is_empty() {
                // First expansion is empty, but we need to recursively expand.
                res.push(VARIABLE_EXPAND_EMPTY);
            }
        }

        // Append all entries in var_item_list, separated by the delimiter.
        res.push_utfstr(&join_strings(&var_item_list, delimit));
        res.push_utfstr(&instr[var_name_and_slice_stop..]);
        return expand_variables(res, out, varexp_char_idx, vars, errors);
    } else {
        // Normal cartesian-product expansion.
        for item in var_item_list {
            if varexp_char_idx == 0 && var_name_and_slice_stop == instr.len() {
                if !out.add(item) {
                    return append_overflow_error(errors, None);
                }
            } else {
                let mut new_in = instr[..varexp_char_idx].to_owned();
                if !new_in.is_empty() {
                    if new_in.as_char_slice().last() != Some(&VARIABLE_EXPAND) {
                        new_in.push(INTERNAL_SEPARATOR);
                    } else if item.is_empty() {
                        new_in.push(VARIABLE_EXPAND_EMPTY);
                    }
                }
                new_in.push_utfstr(&item);
                new_in.push_utfstr(&instr[var_name_and_slice_stop..]);
                let res = expand_variables(new_in, out, varexp_char_idx, vars, errors);
                if res.result != ExpandResultCode::Ok {
                    return res;
                }
            }
        }
    }

    ExpandResult::ok()
}

/// Perform brace expansion, placing the expanded strings into `out`.
fn expand_braces(
    input: WString,
    flags: ExpandFlags,
    out: &mut CompletionReceiver,
    errors: &mut Option<&mut ParseErrorList>,
) -> ExpandResult {
    let mut syntax_error = false;
    let mut brace_count = 0;

    let mut brace_begin = None;
    let mut brace_end = None;
    let mut last_sep = None;

    // Locate the first non-nested brace pair.
    for (pos, c) in input.chars().enumerate() {
        match c {
            BRACE_BEGIN => {
                if brace_count == 0 {
                    brace_begin = Some(pos);
                }
                brace_count += 1;
            }
            BRACE_END => {
                brace_count -= 1;
                #[allow(clippy::comparison_chain)]
                if brace_count < 0 {
                    syntax_error = true;
                } else if brace_count == 0 {
                    brace_end = Some(pos);
                }
            }
            BRACE_SEP if brace_count == 1 => {
                last_sep = Some(pos);
            }
            _ => {
                // we ignore all other characters here
            }
        }
    }

    if brace_count > 0 {
        if !flags.contains(ExpandFlags::FOR_COMPLETIONS) {
            syntax_error = true;
        } else {
            // The user hasn't typed an end brace yet; make one up and append it, then expand
            // that.
            let mut synth = WString::new();
            if let Some(last_sep) = last_sep {
                synth.push_utfstr(&input[..=brace_begin.unwrap()]);
                synth.push_utfstr(&input[last_sep + 1..]);
                synth.push(BRACE_END);
            } else {
                synth.push_utfstr(&input);
                synth.push(BRACE_END);
            }

            return expand_braces(synth, ExpandFlags::FAIL_ON_CMDSUBST, out, errors);
        }
    }

    if syntax_error {
        append_syntax_error!(errors, SOURCE_LOCATION_UNKNOWN, "Mismatched braces");
        return ExpandResult::make_error(STATUS_EXPAND_ERROR);
    }

    let Some(brace_begin) = brace_begin else {
        // No more brace expansions left; restore spaces that were protected while
        // expanding braces and return the value as-is.
        let mut input = input;
        for c in input.as_char_slice_mut() {
            if *c == BRACE_SPACE {
                *c = ' ';
            }
        }
        if !out.add(input) {
            return append_overflow_error(errors, None);
        }
        return ExpandResult::ok();
    };
    let brace_end = brace_end.unwrap();

    let length_preceding_braces = brace_begin;
    let length_following_braces = input.len() - brace_end - 1;
    let tot_len = length_preceding_braces + length_following_braces;
    let mut item_begin = brace_begin + 1;
    for (pos, c) in input.chars().enumerate().skip(brace_begin + 1) {
        if brace_count == 0 && (c == BRACE_SEP || pos == brace_end) {
            assert!(pos >= item_begin);
            let item_len = pos - item_begin;
            let item = &input[item_begin..pos];
            let item = trim(item, Some(wstr::from_char_slice(&[BRACE_SPACE, '\0'])));

            // `whole_item` is a whitespace- and brace-stripped member of a single pass of brace
            // expansion, e.g. in `{ alpha , b,{c, d }}`, `alpha`, `b`, and `c, d` will, in the
            // first round of expansion, each in turn be a `whole_item` (with recursive commas
            // replaced by special placeholders).
            // We recursively call `expand_braces` with each item until it's been fully expanded.
            let mut whole_item = WString::new();
            whole_item.reserve(tot_len + item_len + 2);
            whole_item.push_utfstr(&input[..length_preceding_braces]);
            whole_item.push_utfstr(&item);
            whole_item.push_utfstr(&input[brace_end + 1..]);
            let _ = expand_braces(whole_item, flags, out, errors);

            item_begin = pos + 1;
            if pos == brace_end {
                break;
            }
        }

        if c == BRACE_BEGIN {
            brace_count += 1;
        }

        if c == BRACE_END {
            brace_count -= 1;
        }
    }

    ExpandResult::ok()
}

/// Expand a command substitution `input`, executing on `ctx`, and inserting the results into
/// `out_list`, or any errors into `errors`. Return an expand result.
pub fn expand_cmdsubst(
    input: WString,
    ctx: &mut OperationContext,
    out: &mut CompletionReceiver,
    errors: &mut Option<&mut ParseErrorList>,
) -> ExpandResult {
    let mut cursor = 0;
    let mut is_quoted = false;
    let mut has_dollar = false;
    let cmdsub = match locate_cmdsubst_range(
        &input,
        &mut cursor,
        false,
        Some(&mut is_quoted),
        Some(&mut has_dollar),
    ) {
        Err(()) => {
            append_syntax_error!(errors, SOURCE_LOCATION_UNKNOWN, "Mismatched parenthesis");
            return ExpandResult::make_error(STATUS_EXPAND_ERROR);
        }
        Ok(None) => {
            if !out.add(input) {
                return append_overflow_error(errors, None);
            }
            return ExpandResult::ok();
        }
        Ok(Some(cmdsub)) => cmdsub,
    };

    let mut sub_res = vec![];
    let job_group = ctx.job_group.clone();
    let subshell_status = exec_subshell_for_expand(
        &input[cmdsub.command_range()],
        ctx.parser(),
        job_group.as_ref(),
        &mut sub_res,
    );

    if let Err(subshell_status) = subshell_status {
        // TODO: Ad-hoc switch, how can we enumerate the possible errors more safely?
        let err = match subshell_status {
            _ if subshell_status == STATUS_READ_TOO_MUCH => {
                wgettext!("Too much data emitted by command substitution so it was discarded")
            }
            // TODO: STATUS_CMD_ERROR is overused and too generic. We shouldn't have to test things
            // to figure out what error to show after we've already been given an error code.
            _ if subshell_status == STATUS_CMD_ERROR => {
                if ctx.parser().is_eval_depth_exceeded() {
                    wgettext!("Unable to evaluate string substitution")
                } else {
                    wgettext!("Too many active file descriptors")
                }
            }
            _ if subshell_status == STATUS_CMD_UNKNOWN => {
                wgettext!("Unknown command")
            }
            _ if subshell_status == STATUS_ILLEGAL_CMD => {
                wgettext!("Commandname was invalid")
            }
            _ if subshell_status == STATUS_NOT_EXECUTABLE => {
                wgettext!("Command not executable")
            }
            _ if subshell_status == STATUS_INVALID_ARGS => {
                // TODO: Also overused
                // This is sent for:
                // invalid redirections or pipes (like `<&foo`),
                // invalid variables (invalid name or read-only) for for-loops,
                // switch $foo if $foo expands to more than one argument
                // time in a background job.
                wgettext!("Invalid arguments")
            }
            _ if subshell_status == STATUS_EXPAND_ERROR => {
                // Sent in `for $foo in ...` if $foo expands to more than one word
                wgettext!("Expansion error")
            }
            _ if subshell_status == STATUS_UNMATCHED_WILDCARD => {
                // Sent in `for $foo in ...` if $foo expands to more than one word
                wgettext!("Unmatched wildcard")
            }
            _ => {
                wgettext!("Unknown error while evaluating command substitution")
            }
        };
        append_cmdsub_error!(errors, cmdsub.opening_paren_offset(), cmdsub.end() - 1, err);
        return ExpandResult::make_error(subshell_status);
    }

    // Expand slices like (cat /var/words)[1]
    let mut tail_begin = cmdsub.end();
    if input.as_char_slice().get(tail_begin) == Some(&'[') {
        let mut slice_idx = vec![];
        let slice_begin = tail_begin;
        let slice_end = match parse_slice(&input[slice_begin..], &mut slice_idx, sub_res.len()) {
            Ok(offset) => slice_begin + offset,
            Err((bad_pos, error)) => {
                match error {
                    ParseSliceError::ZeroIndex => {
                        append_syntax_error!(
                            errors,
                            slice_begin + bad_pos,
                            "array indices start at 1, not 0."
                        );
                    }
                    ParseSliceError::InvalidIndex => {
                        append_syntax_error!(errors, slice_begin + bad_pos, "Invalid index value");
                    }
                }
                return ExpandResult::make_error(STATUS_EXPAND_ERROR);
            }
        };

        let mut sub_res2 = vec![];
        tail_begin = slice_end;
        for idx in slice_idx {
            if idx as usize > sub_res.len() || idx < 1 {
                continue;
            }
            // -1 to convert from 1-based slice index to 0-based vector index.
            sub_res2.push(sub_res[idx as usize - 1].clone());
        }
        sub_res = sub_res2;
    }

    // Recursively call ourselves to expand any remaining command substitutions. The result of this
    // recursive call using the tail of the string is inserted into the tail_expand array list
    let mut tail_expand_recv = out.subreceiver();
    let mut tail = input[tail_begin..].to_owned();
    // A command substitution inside double quotes magically closes the quoted string.
    // Reopen the quotes just after the command substitution.
    if is_quoted {
        tail.insert(0, '"');
    }

    let _ = expand_cmdsubst(tail, ctx, &mut tail_expand_recv, errors); // TODO: offset error locations
    let tail_expand = tail_expand_recv.take();

    // Combine the result of the current command substitution with the result of the recursive tail
    // expansion.

    if is_quoted {
        // Awkwardly reconstruct the command output.
        let approx_size = sub_res.iter().map(|sub_item| sub_item.len() + 1).sum();
        let mut sub_res_joined = WString::new();
        sub_res_joined.reserve(approx_size);
        for line in sub_res {
            sub_res_joined.push_utfstr(&escape_string_for_double_quotes(&line));
            sub_res_joined.push('\n');
        }
        // Mimic POSIX shells by stripping all trailing newlines.
        if !sub_res_joined.is_empty() {
            let mut i = sub_res_joined.len();
            while i > 0 && sub_res_joined.as_char_slice()[i - 1] == '\n' {
                i -= 1;
            }
            sub_res_joined.truncate(i);
        }
        // Instead of performing cartesian product expansion, we directly insert the command
        // substitution output into the current expansion results.
        for tail_item in tail_expand {
            let mut whole_item = WString::new();
            whole_item.reserve(
                cmdsub.opening_paren_offset()
                    + 1
                    + sub_res_joined.len()
                    + 1
                    + tail_item.completion.len(),
            );
            whole_item.push_utfstr(
                &input[..cmdsub.opening_paren_offset() - if has_dollar { 1 } else { 0 }],
            );
            whole_item.push(INTERNAL_SEPARATOR);
            whole_item.push_utfstr(&sub_res_joined);
            whole_item.push(INTERNAL_SEPARATOR);
            whole_item.push_utfstr(&tail_item.completion["\"".len()..]);
            if !out.add(whole_item) {
                return append_overflow_error(errors, None);
            }
        }

        return ExpandResult::ok();
    }

    for sub_item in sub_res {
        let sub_item2 = escape_string(&sub_item, EscapeStringStyle::Script(EscapeFlags::COMMA));
        for tail_item in &*tail_expand {
            let mut whole_item = WString::new();
            whole_item.reserve(
                cmdsub.opening_paren_offset()
                    + 1
                    + sub_item2.len()
                    + 1
                    + tail_item.completion.len(),
            );
            whole_item.push_utfstr(
                &input[..cmdsub.opening_paren_offset() - if has_dollar { 1 } else { 0 }],
            );
            whole_item.push(INTERNAL_SEPARATOR);
            whole_item.push_utfstr(&sub_item2);
            whole_item.push(INTERNAL_SEPARATOR);
            whole_item.push_utfstr(&tail_item.completion);
            if !out.add(whole_item) {
                return append_overflow_error(errors, None);
            }
        }
    }

    ExpandResult::ok()
}

// Given that input[0] is HOME_DIRECTORY or tilde (ugh), return the user's name. Return the empty
// string if it is just a tilde. Also return by reference the index of the first character of the
// remaining part of the string (e.g. the subsequent slash).
fn get_home_directory_name<'a>(input: &'a wstr, out_tail_idx: &mut usize) -> &'a wstr {
    assert!([HOME_DIRECTORY, '~'].contains(&input.as_char_slice()[0]));
    // We get the position of the /, but we need to remove it as well.
    if let Some(pos) = input.chars().position(|c| c == '/') {
        *out_tail_idx = pos;
        &input[1..pos]
    } else {
        *out_tail_idx = input.len();
        &input[1..]
    }
}

/// Attempts tilde expansion of the string specified, modifying it in place.
fn expand_home_directory(input: &mut WString, vars: &dyn Environment) {
    let starts_with_tilde = input.as_char_slice().first() == Some(&HOME_DIRECTORY);
    *input = input.replace([HOME_DIRECTORY], L!("~"));
    if !starts_with_tilde {
        return;
    }

    let mut tail_idx = usize::MAX;
    let username = get_home_directory_name(input, &mut tail_idx);
    let mut home = None;
    if username.is_empty() {
        // Current users home directory.
        match vars.get_unless_empty(L!("HOME")) {
            None => {
                input.clear();
                return;
            }
            Some(home_var) => {
                home = Some(home_var.as_string());
                tail_idx = 1;
            }
        }
    } else {
        // POSIX-conforming usernames are ASCII-only
        if let Ok(Some(userinfo)) = User::from_name(&username.to_string()) {
            home = Some(osstr2wcstring(userinfo.dir));
        }
    }

    if let Some(home) = home {
        input.replace_range(..tail_idx, &normalize_path(&home, true));
    }
}

/// Expand the %self escape. Note this can only come at the beginning of the string.
fn expand_percent_self(input: &mut WString) {
    if input.as_char_slice().first() == Some(&PROCESS_EXPAND_SELF) {
        input.replace_range(0..1, &getpid().as_raw().to_wstring());
    }
}

/// Remove any internal separators. Also optionally convert wildcard characters to regular
/// equivalents. This is done to support skip_wildcards.
fn remove_internal_separator(s: &mut WString, conv: bool) {
    // Remove all instances of INTERNAL_SEPARATOR.
    s.retain(|c| c != INTERNAL_SEPARATOR);

    // If conv is true, replace all instances of ANY_STRING with '*',
    // ANY_STRING_RECURSIVE with '*'.
    if conv {
        for idx in s.as_char_slice_mut() {
            match *idx {
                ANY_CHAR => {
                    *idx = '?';
                }
                ANY_STRING | ANY_STRING_RECURSIVE => {
                    *idx = '*';
                }
                _ => {
                    // we ignore all other characters
                }
            }
        }
    }
}

/// A type that knows how to perform expansions.
struct Expander<'a, 'b, 'c> {
    /// Operation context for this expansion.
    ctx: &'c mut OperationContext<'b>,

    /// Flags to use during expansion.
    flags: ExpandFlags,

    /// List to receive any errors generated during expansion, or null to ignore errors.
    errors: &'c mut Option<&'a mut ParseErrorList>,
}

impl<'a, 'b, 'c> Expander<'a, 'b, 'c> {
    fn new(
        ctx: &'c mut OperationContext<'b>,
        flags: ExpandFlags,
        errors: &'c mut Option<&'a mut ParseErrorList>,
    ) -> Self {
        Self { ctx, flags, errors }
    }

    fn expand_string(
        input: WString,
        out_completions: &'a mut CompletionReceiver,
        flags: ExpandFlags,
        ctx: &'a mut OperationContext<'b>,
        mut errors: Option<&'a mut ParseErrorList>,
    ) -> ExpandResult {
        assert!(
            flags.contains(ExpandFlags::FAIL_ON_CMDSUBST) || ctx.has_parser(),
            "Must have a parser if not skipping command substitutions"
        );
        // Early out. If we're not completing, and there's no magic in the input, we're done.
        if !flags.contains(ExpandFlags::FOR_COMPLETIONS) && expand_is_clean(&input) {
            if !out_completions.add(input) {
                return append_overflow_error(&mut errors, None);
            }
            return ExpandResult::ok();
        }

        let mut expand = Expander::new(ctx, flags, &mut errors);

        // Our expansion stages.
        // An expansion stage is a member function pointer.
        // It accepts the input string (transferring ownership) and returns the list of output
        // completions by reference. It may return an error, which halts expansion.
        let stages = [
            Expander::stage_cmdsubst,
            Expander::stage_variables,
            Expander::stage_braces,
            Expander::stage_home_and_self,
            Expander::stage_wildcards,
        ];

        // Load up our single initial completion.
        let mut completions = vec![Completion::from_completion(input.clone())];

        let mut total_result = ExpandResult::ok();
        let mut output_storage = out_completions.subreceiver();
        for stage in stages {
            for comp in completions {
                if expand.ctx.check_cancel() {
                    total_result = ExpandResult::new(ExpandResultCode::Cancel);
                    break;
                }
                let this_result = (stage)(&mut expand, comp.completion, &mut output_storage);
                total_result = this_result;
                if matches!(
                    total_result.result,
                    ExpandResultCode::Error | ExpandResultCode::Overflow
                ) {
                    break;
                }
            }

            // Output becomes our next stage's input.
            completions = output_storage.take();
            if matches!(
                total_result.result,
                ExpandResultCode::Error | ExpandResultCode::Overflow
            ) {
                break;
            }
        }

        // This is a little tricky: if one wildcard failed to match but we still got output, it
        // means that a previous expansion resulted in multiple strings. For example:
        //   set dirs ./a ./b
        //   echo $dirs/*.txt
        // Here if ./a/*.txt matches and ./b/*.txt does not, then we don't want to report a failed
        // wildcard. So swallow failed-wildcard errors if we got any output.
        if total_result == ExpandResultCode::WildcardNoMatch && !completions.is_empty() {
            total_result = ExpandResult::ok();
        }

        if total_result == ExpandResultCode::Ok {
            // Unexpand tildes if we want to preserve them (see #647).
            if flags.contains(ExpandFlags::PRESERVE_HOME_TILDES) {
                expand.unexpand_tildes(&input, &mut completions);
            }
            if !out_completions.extend(completions) {
                total_result = append_overflow_error(expand.errors, None);
            }
        }

        total_result
    }

    fn stage_cmdsubst(&mut self, input: WString, out: &mut CompletionReceiver) -> ExpandResult {
        if self.flags.contains(ExpandFlags::SKIP_CMDSUBST) {
            if !out.add(input) {
                return append_overflow_error(self.errors, None);
            }
            return ExpandResult::ok();
        }
        if self.flags.contains(ExpandFlags::FAIL_ON_CMDSUBST) {
            let mut cursor = 0;
            match locate_cmdsubst_range(&input, &mut cursor, true, None, None) {
                Err(()) => ExpandResult::make_error(STATUS_EXPAND_ERROR),
                Ok(None) => {
                    if !out.add(input) {
                        return append_overflow_error(self.errors, None);
                    }
                    ExpandResult::ok()
                }
                Ok(Some(cmdsub)) => {
                    append_cmdsub_error!(
                        self.errors,
                        cmdsub.opening_paren_offset(),
                        cmdsub.end() - 1,
                        wgettext!(
                            "command substitutions not allowed in command position. Try var=(your-cmd) $var ..."
                        )
                    );
                    ExpandResult::make_error(STATUS_EXPAND_ERROR)
                }
            }
        } else {
            assert!(
                self.ctx.has_parser(),
                "Must have a parser to expand command substitutions"
            );
            expand_cmdsubst(input, self.ctx, out, self.errors)
        }
    }

    // We pass by value to match other stages. NOLINTNEXTLINE(performance-unnecessary-value-param)
    fn stage_variables(&mut self, input: WString, out: &mut CompletionReceiver) -> ExpandResult {
        // We accept incomplete strings here, since complete uses expand_string to expand incomplete
        // strings from the commandline.
        let mut next = unescape_string(
            &input,
            UnescapeStringStyle::Script(UnescapeFlags::SPECIAL | UnescapeFlags::INCOMPLETE),
        )
        .unwrap_or_default();

        if self.flags.contains(ExpandFlags::SKIP_VARIABLES) {
            for i in next.as_char_slice_mut() {
                if [VARIABLE_EXPAND, VARIABLE_EXPAND_SINGLE].contains(i) {
                    *i = '$';
                }
            }
            if !out.add(next) {
                return append_overflow_error(self.errors, None);
            }
            ExpandResult::ok()
        } else {
            let size = next.len();
            expand_variables(next, out, size, self.ctx.vars(), self.errors)
        }
    }

    fn stage_braces(&mut self, input: WString, out: &mut CompletionReceiver) -> ExpandResult {
        expand_braces(input, self.flags, out, self.errors)
    }

    fn stage_home_and_self(
        &mut self,
        mut input: WString,
        out: &mut CompletionReceiver,
    ) -> ExpandResult {
        remove_internal_separator(&mut input, self.flags.contains(ExpandFlags::SKIP_WILDCARDS));

        expand_home_directory(&mut input, self.ctx.vars());
        if !feature_test(FeatureFlag::RemovePercentSelf) {
            expand_percent_self(&mut input);
        }
        if !out.add(input) {
            return append_overflow_error(self.errors, None);
        }
        ExpandResult::ok()
    }

    fn stage_wildcards(
        &mut self,
        path_to_expand: WString,
        out: &mut CompletionReceiver,
    ) -> ExpandResult {
        let mut result = ExpandResult::ok();

        let has_wildcard = wildcard_has_internal(&path_to_expand); // e.g. ANY_STRING
        let for_completions = self.flags.contains(ExpandFlags::FOR_COMPLETIONS);
        let skip_wildcards = self.flags.contains(ExpandFlags::SKIP_WILDCARDS);

        if has_wildcard && self.flags.contains(ExpandFlags::EXECUTABLES_ONLY) {
            // don't do wildcard expansion for executables, see issue #785
        } else if (for_completions && !skip_wildcards) || has_wildcard {
            // We either have a wildcard, or we don't have a wildcard but we're doing completion
            // expansion (so we want to get the completion of a file path). Note that if
            // skip_wildcards is set, we stomped wildcards in remove_internal_separator above, so
            // there actually aren't any.
            //
            // So we're going to treat this input as a file path. Compute the "working directories",
            // which may be CDPATH if the special flag is set.
            let working_dir = self.ctx.vars().get_pwd_slash();
            let mut effective_working_dirs = vec![];
            let for_cd = self.flags.contains(ExpandFlags::SPECIAL_FOR_CD);
            let for_command = self.flags.contains(ExpandFlags::SPECIAL_FOR_COMMAND);
            if !for_cd && !for_command {
                // Common case.
                effective_working_dirs.push(working_dir);
            } else {
                // Either special_for_command or special_for_cd. We can handle these
                // mostly the same. There's the following differences:
                //
                // 1. An empty CDPATH should be treated as '.', but an empty PATH should be left empty
                // (no commands can be found). Also, an empty element in either is treated as '.' for
                // consistency with POSIX shells. Note that we rely on the latter by having called
                // `munge_colon_delimited_array()` for these special env vars. Thus we do not
                // special-case them here.
                //
                // 2. PATH is only "one level," while CDPATH is multiple levels. That is, input like
                // 'foo/bar' should resolve against CDPATH, but not PATH.
                //
                // In either case, we ignore the path if we start with ./ or /. Also ignore it if we are
                // doing command completion and we contain a slash, per IEEE 1003.1, chapter 8 under
                // PATH.
                if path_to_expand.starts_with(L!("/"))
                    || path_to_expand.starts_with(L!("./"))
                    || path_to_expand.starts_with(L!("../"))
                    || (for_command && path_to_expand.contains('/'))
                {
                    effective_working_dirs.push(working_dir);
                } else {
                    // Get the PATH/CDPATH and CWD. Perhaps these should be passed in. An empty CDPATH
                    // implies just the current directory, while an empty PATH is left empty.
                    let mut paths = self
                        .ctx
                        .vars()
                        .get(if for_cd { L!("CDPATH") } else { L!("PATH") })
                        .map(|var| var.as_list().to_owned())
                        .unwrap_or_default();

                    // The current directory is always valid.
                    paths.push(if for_cd { L!(".") } else { L!("") }.to_owned());
                    for next_path in paths {
                        effective_working_dirs
                            .push(path_apply_working_directory(&next_path, &working_dir));
                    }
                }
            }

            result = ExpandResult::new(ExpandResultCode::WildcardNoMatch);
            let mut expanded_recv = out.subreceiver();
            for effective_working_dir in effective_working_dirs {
                let expand_res = wildcard_expand_string(
                    &path_to_expand,
                    &effective_working_dir,
                    self.flags,
                    &*self.ctx.cancel_checker,
                    &mut expanded_recv,
                );
                match expand_res {
                    WildcardResult::Match => result = ExpandResult::ok(),
                    WildcardResult::NoMatch => (),
                    WildcardResult::Overflow => return append_overflow_error(self.errors, None),
                    WildcardResult::Cancel => return ExpandResult::new(ExpandResultCode::Cancel),
                }
            }

            let mut expanded = expanded_recv.take();
            expanded.sort_by(|a, b| wcsfilecmp_glob(&a.completion, &b.completion));
            if !out.extend(expanded) {
                result = ExpandResult::new(ExpandResultCode::Overflow);
            }
        } else {
            // Can't fully justify this check. I think it's that SKIP_WILDCARDS is used when completing
            // to mean don't do file expansions, so if we're not doing file expansions, just drop this
            // completion on the floor.
            #[allow(clippy::collapsible_if)]
            if !for_completions {
                if !out.add(path_to_expand) {
                    return append_overflow_error(self.errors, None);
                }
            }
        }
        result
    }

    // Given an original input string, if it starts with a tilde, "unexpand" the expanded home
    // directory.  Note this may be just a tilde or a user name like ~foo/.
    fn unexpand_tildes(&self, input: &wstr, completions: &mut CompletionList) {
        // If input begins with tilde, then try to replace the corresponding string in each completion
        // with the tilde. If it does not, there's nothing to do.
        if input.as_char_slice().first() != Some(&'~') {
            return;
        }

        // This is a subtle kludge. We need to decide whether to unexpand tildes for all
        // completions, or only those which replace their tokens. The problem is that we're sloppy
        // about setting the COMPLETE_REPLACES_TOKEN flag, except when we're completing in the
        // wildcard stage, because no other clients of string expansion care. Example:
        //   HOME=/foo
        //   mkdir ~/foo # makes /foo/foo
        //   cd ~/<tab>
        // Here we are likely to get a completion 'foo' which may match $HOME, but it extends its token
        // instead of replacing it, so we don't modify it (it will just be appended to the original ~/).
        //
        // However if we are not completing, just expanding, then expansion just produces the full paths
        // so we should unconditionally unexpand tildes.
        let only_replacers = self.flags.contains(ExpandFlags::FOR_COMPLETIONS);

        // Helper to decide whether to process a completion.
        let should_process = |c: &Completion| !only_replacers || c.replaces_token();

        // Early out if none qualify.
        if !completions.iter().any(should_process) {
            return;
        }

        // Get the username_with_tilde (like ~bert) and expand it into a home directory.
        let mut tail_idx = usize::MAX;
        let username_with_tilde =
            L!("~").to_owned() + get_home_directory_name(input, &mut tail_idx);
        let mut home = username_with_tilde.clone();
        expand_tilde(&mut home, self.ctx.vars());

        // Now for each completion that starts with home, replace it with the username_with_tilde.
        for comp in completions {
            if should_process(comp) && comp.completion.starts_with(&home) {
                comp.completion
                    .replace_range(..home.len(), &username_with_tilde);

                // And mark that our tilde is literal, so it doesn't try to escape it.
                comp.flags |= CompleteFlags::DONT_ESCAPE_TILDES;
            }
        }
    }
}

#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub enum ExpandResultCode {
    /// There was an error, for example, unmatched braces.
    Error,
    /// Expansion would exceed the maximum number of elements.
    Overflow,
    /// Expansion succeeded.
    Ok,
    /// Expansion was cancelled (e.g. control-C).
    Cancel,
    /// Expansion succeeded, but a wildcard in the string matched no files,
    /// so the output is empty.
    WildcardNoMatch,
}

/// These are the possible return values for expand_string.
#[must_use]
#[derive(Debug)]
pub struct ExpandResult {
    /// The result of expansion.
    pub result: ExpandResultCode,

    /// If expansion resulted in an error, this is an appropriate value with which to populate
    /// $status.
    pub status: libc::c_int,
}

#[cfg(test)]
mod tests {
    use crate::{
        abbrs::{self, Abbreviation, with_abbrs, with_abbrs_mut},
        complete::{CompletionList, CompletionReceiver},
        env::{EnvMode, EnvStackSetResult},
        expand::{ExpandFlags, ExpandResultCode, expand_string, expand_to_receiver},
        operation_context::{EXPANSION_LIMIT_DEFAULT, OperationContext, no_cancel},
        parse_constants::ParseErrorList,
        parser::ParserEnvSetMode,
        prelude::*,
        tests::prelude::*,
    };
    use fish_widestring::{ANY_STRING, str2wcstring};
    use std::collections::{HashMap, HashSet, hash_map::RandomState};

    fn expand_test_impl(
        input: &wstr,
        flags: ExpandFlags,
        expected: Vec<WString>,
        error_message: Option<&str>,
    ) {
        let parser = &mut TestParser::new();
        let mut output = CompletionList::new();
        let mut errors = ParseErrorList::new();
        let pwd = PwdEnvironment::default();
        let ctx = &mut OperationContext::test_only_foreground(parser, &pwd, Box::new(no_cancel));

        if expand_string(input.to_owned(), &mut output, flags, ctx, Some(&mut errors))
            == ExpandResultCode::Error
        {
            assert_ne!(
                errors,
                vec![],
                "Bug: Parse error reported but no error text found."
            );
            panic!(
                "{}",
                errors[0].describe(input, ctx.parser().is_interactive())
            );
        }

        let expected_set: HashSet<WString, RandomState> = HashSet::from_iter(expected);
        let output_set = HashSet::from_iter(output.into_iter().map(|c| c.completion));
        assert_eq!(
            expected_set,
            output_set,
            "{}",
            error_message.unwrap_or("expand mismatch")
        );
    }

    // Test globbing and other parameter expansion.
    #[test]
    #[serial]
    fn test_expand() {
        test_init();
        let TestParser {
            ref mut parser,
            ref mut pushed_dirs,
        } = TestParser::new();
        /// Perform parameter expansion and test if the output equals the zero-terminated parameter list /// supplied.
        ///
        /// \param in the string to expand
        /// \param flags the flags to send to expand_string
        /// \param ... A zero-terminated parameter list of values to test.
        /// After the zero terminator comes one more arg, a string, which is the error
        /// message to print if the test fails.
        macro_rules! expand_test {
            ($input:expr, $flags:expr, ( $($expected:expr),* $(,)? )) => {
                expand_test_impl(L!($input), $flags, vec![$( $expected.into(), )*], None)
            };
            ($input:expr, $flags:expr, ( $($expected:expr),* $(,)? ), $error:literal) => {
                expand_test_impl(L!($input), $flags, vec![$( $expected.into(), )*], Some($error))
            };
            ($input:expr, $flags:expr, $expected:expr) => {
                expand_test_impl(L!($input), $flags, vec![ $expected.into() ], None)
            };
            ($input:expr, $flags:expr, $expected:expr, $error:literal) => {
                expand_test_impl(L!($input), $flags, vec![ $expected.into() ], Some($error))
            };
        }

        // Testing parameter expansion
        let noflags = ExpandFlags::default();

        expand_test!("foo", noflags, "foo", "Strings do not expand to themselves");

        expand_test!(
            "a{b,c,d}e",
            noflags,
            ("abe", "ace", "ade"),
            "Bracket expansion is broken"
        );
        expand_test!(
            "a*",
            ExpandFlags::SKIP_WILDCARDS,
            "a*",
            "Cannot skip wildcard expansion"
        );
        expand_test!(
            "/bin/l\\0",
            ExpandFlags::FOR_COMPLETIONS,
            (),
            "Failed to handle null escape in expansion"
        );
        expand_test!(
            "foo\\$bar",
            ExpandFlags::SKIP_VARIABLES,
            "foo$bar",
            "Failed to handle dollar sign in variable-skipping expansion"
        );

        // bb
        //    x
        // bar
        // baz
        //    xxx
        //    yyy
        // bax
        //    xxx
        // lol
        //    nub
        //       q
        //       zzz
        // .foo
        // aaa
        // aaa2
        //    x
        std::fs::create_dir_all("test/fish_expand_test/").unwrap();
        std::fs::create_dir_all("test/fish_expand_test/bb/").unwrap();
        std::fs::create_dir_all("test/fish_expand_test/baz/").unwrap();
        std::fs::create_dir_all("test/fish_expand_test/bax/").unwrap();
        std::fs::create_dir_all("test/fish_expand_test/lol/nub/").unwrap();
        std::fs::create_dir_all("test/fish_expand_test/aaa/").unwrap();
        std::fs::create_dir_all("test/fish_expand_test/aaa2/").unwrap();
        std::fs::write("test/fish_expand_test/.foo", []).unwrap();
        std::fs::write("test/fish_expand_test/bb/x", []).unwrap();
        std::fs::write("test/fish_expand_test/bar", []).unwrap();
        std::fs::write("test/fish_expand_test/bax/xxx", []).unwrap();
        std::fs::write("test/fish_expand_test/baz/xxx", []).unwrap();
        std::fs::write("test/fish_expand_test/baz/yyy", []).unwrap();
        std::fs::write("test/fish_expand_test/lol/nub/q", []).unwrap();
        std::fs::write("test/fish_expand_test/lol/nub/zzz", []).unwrap();
        std::fs::write("test/fish_expand_test/aaa2/x", []).unwrap();

        // This is checking that .* does NOT match . and ..
        // (https://github.com/fish-shell/fish-shell/issues/270). But it does have to match literal
        // components (e.g. "./*" has to match the same as "*".
        expand_test!(
            "test/fish_expand_test/.*",
            noflags,
            "test/fish_expand_test/.foo",
            "Expansion not correctly handling dotfiles"
        );

        expand_test!(
            "test/fish_expand_test/./.*",
            noflags,
            "test/fish_expand_test/./.foo",
            "Expansion not correctly handling literal path components in dotfiles"
        );

        expand_test!(
            "test/fish_expand_test/*/xxx",
            noflags,
            (
                "test/fish_expand_test/bax/xxx",
                "test/fish_expand_test/baz/xxx"
            ),
            "Glob did the wrong thing 1"
        );

        expand_test!(
            "test/fish_expand_test/*z/xxx",
            noflags,
            "test/fish_expand_test/baz/xxx",
            "Glob did the wrong thing 2"
        );

        expand_test!(
            "test/fish_expand_test/**z/xxx",
            noflags,
            "test/fish_expand_test/baz/xxx",
            "Glob did the wrong thing 3"
        );

        expand_test!(
            "test/fish_expand_test////baz/xxx",
            noflags,
            "test/fish_expand_test////baz/xxx",
            "Glob did the wrong thing 3"
        );

        expand_test!(
            "test/fish_expand_test/b**",
            noflags,
            (
                "test/fish_expand_test/bb",
                "test/fish_expand_test/bb/x",
                "test/fish_expand_test/bar",
                "test/fish_expand_test/bax",
                "test/fish_expand_test/bax/xxx",
                "test/fish_expand_test/baz",
                "test/fish_expand_test/baz/xxx",
                "test/fish_expand_test/baz/yyy"
            ),
            "Glob did the wrong thing 4"
        );

        // A trailing slash should only produce directories.
        expand_test!(
            "test/fish_expand_test/b*/",
            noflags,
            (
                "test/fish_expand_test/bb/",
                "test/fish_expand_test/baz/",
                "test/fish_expand_test/bax/"
            ),
            "Glob did the wrong thing 5"
        );

        expand_test!(
            "test/fish_expand_test/b**/",
            noflags,
            (
                "test/fish_expand_test/bb/",
                "test/fish_expand_test/baz/",
                "test/fish_expand_test/bax/"
            ),
            "Glob did the wrong thing 6"
        );

        expand_test!(
            "test/fish_expand_test/**/q",
            noflags,
            "test/fish_expand_test/lol/nub/q",
            "Glob did the wrong thing 7"
        );

        expand_test!(
            "test/fish_expand_test/BA",
            ExpandFlags::FOR_COMPLETIONS,
            (
                "test/fish_expand_test/bar",
                "test/fish_expand_test/bax/",
                "test/fish_expand_test/baz/"
            ),
            "Case insensitive test did the wrong thing"
        );

        expand_test!(
            "test/fish_expand_test/BA",
            ExpandFlags::FOR_COMPLETIONS,
            (
                "test/fish_expand_test/bar",
                "test/fish_expand_test/bax/",
                "test/fish_expand_test/baz/"
            ),
            "Case insensitive test did the wrong thing"
        );

        expand_test!(
            "test/fish_expand_test/bb/yyy",
            ExpandFlags::FOR_COMPLETIONS,
            (), /* nothing! */
            "Wrong fuzzy matching 1"
        );

        expand_test!(
            "test/fish_expand_test/bb/x",
            ExpandFlags::FOR_COMPLETIONS | ExpandFlags::FUZZY_MATCH,
            "",
            // we just expect the empty string since this is an exact match
            "Wrong fuzzy matching 2"
        );

        // Some vswprintfs refuse to append ANY_STRING in a format specifiers, so don't use
        // format_string here.
        let fuzzy_comp = ExpandFlags::FOR_COMPLETIONS | ExpandFlags::FUZZY_MATCH;
        let any_str_str = ANY_STRING.to_string();
        expand_test!(
            "test/fish_expand_test/b/xx*",
            fuzzy_comp,
            (
                (String::from("test/fish_expand_test/bax/xx") + &any_str_str),
                (String::from("test/fish_expand_test/baz/xx") + &any_str_str)
            ),
            "Wrong fuzzy matching 3"
        );

        expand_test!(
            "test/fish_expand_test/*/nu/zz",
            fuzzy_comp,
            (format!("test/fish_expand_test/{any_str_str}/nub/zzz")),
            "Glob did not expand correctly with more than one path item after the *"
        );

        expand_test!(
            "test/fish_expand_test/b/yyy",
            fuzzy_comp,
            "test/fish_expand_test/baz/yyy",
            "Wrong fuzzy matching 4"
        );

        expand_test!(
            "test/fish_expand_test/aa/x",
            fuzzy_comp,
            "test/fish_expand_test/aaa2/x",
            "Wrong fuzzy matching 5"
        );

        expand_test!(
            "test/fish_expand_test/aaa/x",
            fuzzy_comp,
            (),
            "Wrong fuzzy matching 6 - shouldn't remove valid directory names (#3211)"
        );

        // Dotfiles
        expand_test!(
            "test/fish_expand_test/.*",
            noflags,
            "test/fish_expand_test/.foo",
            ""
        );

        // Literal path components in dotfiles.
        expand_test!(
            "test/fish_expand_test/./.*",
            noflags,
            "test/fish_expand_test/./.foo",
            ""
        );

        parser.pushd(pushed_dirs, "test/fish_expand_test");

        expand_test!(
            "b/xx",
            fuzzy_comp,
            ("bax/xxx", "baz/xxx"),
            "Wrong fuzzy matching 5"
        );

        // multiple slashes with fuzzy matching - #3185
        expand_test!("l///n", fuzzy_comp, "lol///nub/", "Wrong fuzzy matching 6");

        parser.popd(pushed_dirs);
    }

    #[test]
    #[serial]
    fn test_expand_overflow() {
        test_init();
        // Testing overflowing expansions
        // Ensure that we have sane limits on number of expansions - see #7497.

        // Make a list of 64 elements, then expand it cartesian-style 64 times.
        // This is far too large to expand.
        let vals: Vec<WString> = (1..=64).map(|i| i.to_wstring()).collect();
        let expansion = str2wcstring(str::repeat("$bigvar", 64));

        let parser = &mut TestParser::new();
        parser.vars().push(true);
        let set = parser.set_var(L!("bigvar"), ParserEnvSetMode::new(EnvMode::LOCAL), vals);
        assert_eq!(set, EnvStackSetResult::Ok);

        let mut errors = ParseErrorList::new();
        let ctx =
            &mut OperationContext::foreground(parser, Box::new(no_cancel), EXPANSION_LIMIT_DEFAULT);

        // We accept only 1024 completions.
        let mut output = CompletionReceiver::new(1024);

        let res = expand_to_receiver(
            expansion,
            &mut output,
            ExpandFlags::default(),
            ctx,
            Some(&mut errors),
        );
        assert_ne!(errors, vec![]);
        assert_eq!(res, ExpandResultCode::Error);

        ctx.parser().vars().pop(false);
    }

    #[test]
    #[serial]
    fn test_abbreviations() {
        test_init();
        // Testing abbreviations

        with_abbrs_mut(|abbrset| {
            abbrset.add(Abbreviation::new(
                L!("gc").to_owned(),
                L!("gc").to_owned(),
                L!("git checkout").to_owned(),
                abbrs::Position::Command,
                false,
            ));
            abbrset.add(Abbreviation::new(
                L!("foo").to_owned(),
                L!("foo").to_owned(),
                L!("bar").to_owned(),
                abbrs::Position::Command,
                false,
            ));
            abbrset.add(Abbreviation::new(
                L!("gx").to_owned(),
                L!("gx").to_owned(),
                L!("git checkout").to_owned(),
                abbrs::Position::Command,
                false,
            ));
            abbrset.add(Abbreviation::new(
                L!("yin").to_owned(),
                L!("yin").to_owned(),
                L!("yang").to_owned(),
                abbrs::Position::Anywhere,
                false,
            ));
        });

        // Helper to expand an abbreviation, enforcing we have no more than one result.
        let abbr_expand_1 = |token, pos| -> Option<WString> {
            let result = with_abbrs(|abbrset| abbrset.r#match(token, pos, L!("")));
            if result.is_empty() {
                return None;
            }
            assert_eq!(
                &result[1..],
                &[],
                "abbreviation expansion for {token} returned more than 1 result"
            );
            Some(result.into_iter().next().unwrap().replacement)
        };

        let cmd = abbrs::Position::Command;
        assert!(
            abbr_expand_1(L!(""), cmd).is_none(),
            "Unexpected success with empty abbreviation"
        );
        assert!(
            abbr_expand_1(L!("nothing"), cmd).is_none(),
            "Unexpected success with missing abbreviation"
        );

        assert_eq!(
            abbr_expand_1(L!("gc"), cmd),
            Some(L!("git checkout").into())
        );

        assert_eq!(abbr_expand_1(L!("foo"), cmd), Some(L!("bar").into()));

        with_abbrs_mut(|abbrset| abbrset.clear());
    }

    #[test]
    fn test_replace_home_directory_with_tilde() {
        use super::replace_home_directory_with_tilde as rhdwt;
        let vars = TestEnvironment {
            vars: HashMap::from([(L!("HOME").to_owned(), L!("/home/testuser").to_owned())]),
        };

        assert_eq!(rhdwt("/home/testuser/", &vars), "~/");
        assert_eq!(rhdwt("/home/testuser/Documents/", &vars), "~/Documents/");
        assert_eq!(rhdwt("/home/testuser", &vars), "/home/testuser");
        assert_eq!(rhdwt("/other/path/", &vars), "/other/path/");
        assert_eq!(rhdwt("relative/path", &vars), "relative/path");
    }
}
