builtin commandline: -x for expanded tokens, supplanting -o

Issue #10194 reports Cobra completions do

    set -l args (commandline -opc)
    eval $args[1] __complete $args[2..] (commandline -ct | string escape)

The intent behind "eval" is to expand variables and tildes in "$args".
Fair enough. Several of our own completions do the same, see the next commit.

The problem with "commandline -o" + "eval" is that the former already
removes quotes that are  relevant for "eval". This becomes a problem if $args
contains quoted () or {}, for example this command will wrongly execute a
command substituion:

    git --work-tree='(launch-missiles)' <TAB>

It is possible to escape the string the tokens before running eval, but
then there will be no expansion of variables etc.  The problem is that
"commandline -o" only unescapes tokens so they end up in a weird state
somewhere in-between what the user typed and the expanded version.

Remove the need for "eval" by introducing "commandline -x" which expands
things like variables and braces. This enables custom completion scripts to
be aware of shell variables without eval, see the added test for completions
to "make -C $var/some/dir ".

This means that essentially all third party scripts should migrate from
"commandline -o" to "commandline -x". For example

    set -l tokens
    if commandline -x >/dev/null 2>&1
        set tokens (commandline -xpc)
    else
        set tokens (commandline -opc)
    end

Since this is mainly used for completions, the expansion skips command
substitutions.  They are passed through as-is (instead of cancelling or
expanding to nothing) to make custom completion scripts work reasonably well
in the common case. Of course there are cases where we would want to expand
command substitutions here, so I'm not sure.

tests/checks/complete.fish

@@ -552,3 +552,44 @@ rm -r $tmpdir
 complete -C'complete --command=mktemp' | string replace -rf '=mktemp\t.*' '=mktemp'
 # (one "--command=" is okay, we used to get "--command=--command="
 # CHECK: --command=mktemp
+
+## Test token expansion in commandline -x
+
+complete complete_make -f -a '(argparse C/directory= -- (commandline -xpc)[2..];
+                               echo Completing targets in directory $_flag_C)'
+var=path/to complete -C'complete_make -C "$var/build-directory" '
+# CHECK: Completing targets in directory path/to/build-directory
+var1=path complete -C'var2=to complete_make -C "$var1/$var2/other-build-directory" '
+# CHECK: Completing targets in directory path/to/other-build-directory
+
+complete complete_existing_argument -f -a '(commandline -xpc)[2..]'
+var=a_value complete -C'complete_existing_argument "1  2" $var \'quoted (foo bar)\' unquoted(baz qux) '
+# CHECK: 1  2
+# CHECK: a_value
+# CHECK: quoted (foo bar)
+# CHECK: unquoted(baz qux)
+
+complete complete_first_argument_and_count -f -a '(set -l args (commandline -xpc)[2..]
+                                        echo (count $args) arguments, first argument is $args[1])'
+list=arg(seq 10) begin
+    complete -C'complete_first_argument_and_count $list$list '
+    # CHECK: 100 arguments, first argument is arg1arg1
+    complete -C'complete_first_argument_and_count $list$list$list '
+    # CHECK: 1 arguments, first argument is $list$list$list
+end
+
+## Test commandline --tokens-raw
+complete complete_raw_tokens -f -ka '(commandline --tokens-raw)'
+complete -C'complete_raw_tokens "foo" bar\\ baz (qux) '
+# CHECK: complete_raw_tokens
+# CHECK: "foo"
+# CHECK: bar\ baz
+# CHECK: (qux)
+
+## Test deprecated commandline -o
+complete complete_unescaped_tokens -f -ka '(commandline -o)'
+complete -C'complete_unescaped_tokens "foo" bar\\ baz (qux) '
+# CHECK: complete_unescaped_tokens
+# CHECK: foo
+# CHECK: bar baz
+# CHECK: (qux)