Release automation script

Things that are not currently happening in this workflow:
- No GPG-signature on the Git tag
- No *.asc signature file for the tarball (or for any other release assets)
- No GPG-signed Debian and other OBS packages

To-do:
- remove the corresponding entries from
  https://github.com/fish-shell/fish-shell/wiki/Release-checklist
  and link to this workflow.
- Maybe add some testing (for the Linux packages)?.
- Let's hope that this doesn't cause security issues.

Usage:
1. run "build_tools/release.sh $version"; this will create and push
   a tag, which kicks off .github/workflows/release.yml
2. wait for the draft release to be created at
   https://github.com/fish-shell/fish-shell/releases/tags/$version
3. publish the draft (manually, for now). This should unblock the
   last part of the workflow (website updates).

Closes #10449

Incremental usage example:

	version=4.0.3
	repository_owner=fish-shell
	remote=origin
	cd ../fish-shell-secondary-worktree
	git tag -d $version ||:
	git push $remote :$version ||:
	git reset --hard origin/Integration_$version
	for d in .github build_tools; do {
		rm -rf $d
		cp -r ../fish-shell/$d .
		git add $d
	} done
	git commit -m 'Backport CI/CD'
	echo "See https://github.com/$repository_owner/fish-shell/actions"
	echo "See the draft release at https://github.com/$repository_owner/fish-shell/releases/$version"
	../fish-shell/build_tools/release.sh $version $repository_owner $remote

.editorconfig

@@ -19,6 +19,9 @@ trim_trailing_whitespace = false
 [*.sh]
 indent_size = 4
 
+[build_tools/release.sh]
+max_line_length = 72
+
 [Dockerfile]
 indent_size = 2
 

.github/workflows/mac_codesign.yml

@@ -1,42 +0,0 @@
-name: macOS build and codesign
-
-on:
-  workflow_dispatch: # Enables manual trigger from GitHub UI
-
-jobs:
-  build-and-code-sign:
-    runs-on: macos-latest
-    environment: macos-codesign
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install Rust
-        uses: ./.github/actions/rust-toolchain@oldest-supported
-        with:
-          targets: x86_64-apple-darwin
-      - name: Install Rust Stable
-        uses: ./.github/actions/rust-toolchain@stable
-        with:
-          targets: aarch64-apple-darwin
-      - name: build-and-codesign
-        run: |
-          export FISH_ARTEFACT_PATH=/tmp/fish-built
-          # macOS runners keep having issues loading Cargo.toml dependencies from git (GitHub) instead
-          # of crates.io, so give this a try. It's also sometimes significantly faster on all platforms.
-          export CARGO_NET_GIT_FETCH_WITH_CLI=true
-          cargo install apple-codesign
-          mkdir -p "$FISH_ARTEFACT_PATH"
-          echo "$MAC_CODESIGN_APP_P12_BASE64" | base64 --decode > /tmp/app.p12
-          echo "$MAC_CODESIGN_INSTALLER_P12_BASE64" | base64 --decode > /tmp/installer.p12
-          echo "$MACOS_NOTARIZE_JSON" > /tmp/notarize.json
-          ./build_tools/make_pkg.sh -s -f /tmp/app.p12 -i /tmp/installer.p12 -p "$MAC_CODESIGN_PASSWORD" -n -j /tmp/notarize.json
-          rm /tmp/installer.p12 /tmp/app.p12 /tmp/notarize.json
-        env:
-          MAC_CODESIGN_APP_P12_BASE64: ${{ secrets.MAC_CODESIGN_APP_P12_BASE64 }}
-          MAC_CODESIGN_INSTALLER_P12_BASE64: ${{ secrets.MAC_CODESIGN_INSTALLER_P12_BASE64 }}
-          MAC_CODESIGN_PASSWORD: ${{ secrets.MAC_CODESIGN_PASSWORD }}
-          MACOS_NOTARIZE_JSON: ${{ secrets.MACOS_NOTARIZE_JSON }}
-      - uses: actions/upload-artifact@v4
-        with:
-          name: macOS Artefacts
-          path: /tmp/fish-built/*
-          if-no-files-found: error

.github/workflows/release.yml

@@ -0,0 +1,211 @@
+name: Create a new release
+
+on:
+  push:
+    tags:
+      - '*.*.*'
+
+permissions:
+  contents: write
+
+jobs:
+  is-release-tag:
+    name: Pre-release checks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Workaround for https://github.com/actions/checkout/issues/882
+          ref: ${{ github.ref }}
+      - name: Check if the pushed tag looks like a release
+        run: |
+          set -x
+          commit_subject=$(git log -1 --format=%s)
+          tag=$(git describe)
+          [ "$commit_subject" = "Release $tag" ]
+
+
+  source-tarball:
+    needs: [is-release-tag]
+    name: Create the source tarball
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+      tarball-name: ${{ steps.version.outputs.tarball-name }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Workaround for https://github.com/actions/checkout/issues/882
+          ref: ${{ github.ref }}
+      - name: Install dependencies
+        run: sudo apt install cmake gettext ninja-build python3-pip python3-sphinx
+      - name: Create tarball
+        run: |
+          set -x
+          mkdir /tmp/fish-built
+          FISH_ARTEFACT_PATH=/tmp/fish-built ./build_tools/make_tarball.sh
+          {
+            pip install sphinx-markdown-builder==0.6.8
+            relnotes_tmp=$(mktemp -d)
+            mkdir "$relnotes_tmp/src" "$relnotes_tmp/out"
+            version=$(git describe)
+            minor_version=${version%.*}
+            # Delete notes for prior releases.
+            # Also fix up any relative references to other documentation files.
+            awk <CHANGELOG.rst '
+              /^fish/ && $2 != "'"$version"'" { exit }
+              { print }
+            ' |
+              sed >"$relnotes_tmp/src"/index.rst \
+                -e 's,:doc:`\(.*\) <\([^>]*\)>`,`\1 <https://fishshell.com/docs/'"$minor_version"'/\2.html>`_,g' \
+                -e 's,:envvar:`\([^`]*\)`,``$\1``,g'
+            # In future, we could reuse doctree from when we made HTML docs.
+            sphinx-build -j 1 $(: "sphinx-markdown-builder is not marked concurrency-safe") \
+              -W -E -b markdown -c doc_src \
+              -d "$relnotes_tmp/doctree" "$relnotes_tmp/src" $relnotes_tmp/out
+            # Delete title
+            sed -n 1p "$relnotes_tmp/out/index.md" | grep -q "^# fish .*"
+            sed -n 2p "$relnotes_tmp/out/index.md" | grep -q '^$'
+            sed -i 1,2d "$relnotes_tmp/out/index.md"
+            {
+              cat "$relnotes_tmp/out/index.md" - <<EOF
+          ----
+
+          *Download links: To download the source code for fish, we suggest the file named "fish-$version.tar.xz". The file downloaded from "Source code (tar.gz)" will not build correctly.*
+
+          *There is no GPG signature because we haven't yet decided how to integrate signing into the new release automation.*
+
+          *The files called fish-$version-linux-\*.tar.xz are experimental packages containing a single standalone ``fish`` binary for any Linux with the given architecture.*
+          EOF
+            } >/tmp/fish-built/release-notes.md
+            rm -r "$relnotes_tmp"
+          }
+      - name: Upload tarball artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: source-tarball
+          path: |
+            /tmp/fish-built/fish-${{ github.ref_name }}.tar.xz
+            /tmp/fish-built/release-notes.md
+          if-no-files-found: error
+
+  packages-for-linux:
+    needs: [is-release-tag]
+    name: Build single-file fish for Linux (experimental)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Workaround for https://github.com/actions/checkout/issues/882
+          ref: ${{ github.ref }}
+      - name: Install Rust Stable
+        uses: ./.github/actions/rust-toolchain@stable
+        with:
+          targets: x86_64-unknown-linux-musl,aarch64-unknown-linux-musl
+      - name: Install dependencies
+        run: sudo apt install crossbuild-essential-arm64 musl-tools python3-sphinx
+      - name: Build statically-linked executables
+        run: |
+          set -x
+          CFLAGS="-D_FORTIFY_SOURCE=2" \
+            CMAKE_WITH_GETTEXT=0 \
+            CC=aarch64-linux-gnu-gcc \
+            RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc -C link-arg=-lgcc -C link-arg=-D_FORTIFY_SOURCE=0" \
+            cargo build --release --target aarch64-unknown-linux-musl --bin fish
+          cargo build --release --target x86_64-unknown-linux-musl --bin fish
+      - name: Compress
+        run: |
+          set -x
+          for arch in x86_64 aarch64; do
+            tar -cazf fish-$(git describe)-linux-$arch.tar.xz \
+              -C target/$arch-unknown-linux-musl/release fish
+          done
+      - uses: actions/upload-artifact@v4
+        with:
+          name: Static builds for Linux
+          path: fish-${{ github.ref_name }}-linux-*.tar.xz
+          if-no-files-found: error
+
+  create-draft-release:
+    needs:
+      - is-release-tag
+      - source-tarball
+      - packages-for-linux
+    name: Create release draft
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Workaround for https://github.com/actions/checkout/issues/882
+          ref: ${{ github.ref }}
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          merge-multiple: true
+          path: /tmp/artifacts
+      - name: List artifacts
+        run: find /tmp/artifacts -type f
+      - name: Create draft release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ github.ref_name }}
+          name: fish ${{ github.ref_name }}
+          body_path: /tmp/artifacts/release-notes.md
+          draft: true
+          files: |
+            /tmp/artifacts/fish-${{ github.ref_name }}.tar.xz # source tarball
+            /tmp/artifacts/fish-${{ github.ref_name }}-linux-*.tar.xz # Linux packages
+
+  packages-for-macos:
+    needs: [is-release-tag, create-draft-release]
+    name: Build packages for macOS
+    runs-on: macos-latest
+    environment: macos-codesign
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Workaround for https://github.com/actions/checkout/issues/882
+          ref: ${{ github.ref }}
+      - name: Install Rust
+        uses: ./.github/actions/rust-toolchain@oldest-supported
+        with:
+          targets: x86_64-apple-darwin
+      - name: Install Rust Stable
+        uses: ./.github/actions/rust-toolchain@stable
+        with:
+          targets: aarch64-apple-darwin
+      - name: Build and codesign
+        run: |
+          die() { echo >&2 "$*"; exit 1; }
+          [ -n "$MAC_CODESIGN_APP_P12_BASE64" ] || die "Missing MAC_CODESIGN_APP_P12_BASE64"
+          [ -n "$MAC_CODESIGN_INSTALLER_P12_BASE64" ] || die "Missing MAC_CODESIGN_INSTALLER_P12_BASE64"
+          [ -n "$MAC_CODESIGN_PASSWORD" ] || die "Missing MAC_CODESIGN_PASSWORD"
+          [ -n "$MACOS_NOTARIZE_JSON" ] || die "Missing MACOS_NOTARIZE_JSON"
+          set -x
+          export FISH_ARTEFACT_PATH=/tmp/fish-built
+          # macOS runners keep having issues loading Cargo.toml dependencies from git (GitHub) instead
+          # of crates.io, so give this a try. It's also sometimes significantly faster on all platforms.
+          export CARGO_NET_GIT_FETCH_WITH_CLI=true
+          cargo install apple-codesign
+          mkdir -p "$FISH_ARTEFACT_PATH"
+          echo "$MAC_CODESIGN_APP_P12_BASE64" | base64 --decode >/tmp/app.p12
+          echo "$MAC_CODESIGN_INSTALLER_P12_BASE64" | base64 --decode >/tmp/installer.p12
+          echo "$MACOS_NOTARIZE_JSON" >/tmp/notarize.json
+          ./build_tools/make_macos_pkg.sh -s -f /tmp/app.p12 \
+            -i /tmp/installer.p12 -p "$MAC_CODESIGN_PASSWORD" \
+            -n -j /tmp/notarize.json
+          [ -f "${FISH_ARTEFACT_PATH}/fish-${{ github.ref_name }}.app.zip" ]
+          [ -f "${FISH_ARTEFACT_PATH}/fish-${{ github.ref_name }}.pkg" ]
+          rm /tmp/installer.p12 /tmp/app.p12 /tmp/notarize.json
+        env:
+          MAC_CODESIGN_APP_P12_BASE64: ${{ secrets.MAC_CODESIGN_APP_P12_BASE64 }}
+          MAC_CODESIGN_INSTALLER_P12_BASE64: ${{ secrets.MAC_CODESIGN_INSTALLER_P12_BASE64 }}
+          MAC_CODESIGN_PASSWORD: ${{ secrets.MAC_CODESIGN_PASSWORD }}
+          MACOS_NOTARIZE_JSON: ${{ secrets.MACOS_NOTARIZE_JSON }}
+      - name: Add macOS packages to the release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release upload $(git describe) \
+            /tmp/fish-built/fish-${{ github.ref_name }}.app.zip \
+            /tmp/fish-built/fish-${{ github.ref_name }}.pkg

.github/workflows/staticbuild.yml

@@ -1,80 +0,0 @@
-name: staticbuilds
-
-on:
-    # release:
-    #     types: [published]
-    # schedule:
-    # - cron: "14 13 * * *"
-    workflow_dispatch:
-
-jobs:
-  staticbuilds-linux:
-
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: read
-
-    steps:
-    - uses: ./.github/actions/rust-toolchain@stable
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-    - name: Prepare
-      run: |
-           sudo apt install python3-sphinx
-           rustup target add x86_64-unknown-linux-musl
-           rustup target add aarch64-unknown-linux-musl
-           sudo apt install musl-tools crossbuild-essential-arm64 python3-pexpect tmux -y
-    - name: Build
-      run: |
-           CFLAGS="$CFLAGS -D_FORTIFY_SOURCE=2" \
-             CMAKE_WITH_GETTEXT=0 \
-             CC=aarch64-linux-gnu-gcc \
-             RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc -C link-arg=-lgcc -C link-arg=-D_FORTIFY_SOURCE=0" \
-             cargo build --release --target aarch64-unknown-linux-musl --bin fish
-           cargo build --release --target x86_64-unknown-linux-musl
-    - name: Test
-      run: |
-           tests/test_driver.py target/x86_64-unknown-linux-musl/release/
-    - name: Compress
-      run: |
-           tar -cazf fish-static-x86_64-$(git describe).tar.xz -C target/x86_64-unknown-linux-musl/release/ fish
-           tar -cazf fish-static-aarch64-$(git describe).tar.xz -C target/aarch64-unknown-linux-musl/release/ fish
-    - uses: actions/upload-artifact@v4
-      with:
-        name: fish-static-linux
-        path: |
-              fish-*.tar.xz
-        retention-days: 14
-  staticbuilds-macos:
-
-    runs-on: macos-latest
-
-    permissions:
-      contents: read
-
-    steps:
-    - uses: ./.github/actions/rust-toolchain@oldest-supported
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-    - name: Prepare
-      run: |
-           sudo pip3 install --break-system-packages sphinx
-           rustup target add x86_64-apple-darwin
-           rustup target add aarch64-apple-darwin
-    - name: Build
-      run: |
-           PCRE2_SYS_STATIC=1 cargo build --release --target aarch64-apple-darwin --bin fish
-           PCRE2_SYS_STATIC=1 cargo build --release --target x86_64-apple-darwin --bin fish
-    - name: Compress
-      run: |
-           tar -cazf fish-macos-aarch64.tar.xz -C target/aarch64-apple-darwin/release/ fish
-           tar -cazf fish-macos-x86_64.tar.xz -C target/x86_64-apple-darwin/release/ fish
-    - uses: actions/upload-artifact@v4
-      with:
-        name: fish-static-macos
-        path: |
-              fish-macos-*.tar.xz
-        retention-days: 14

build_tools/release.sh

@@ -0,0 +1,176 @@
+#!/bin/sh
+
+{
+
+set -ex
+
+version=$1
+repository_owner=fish-shell
+remote=origin
+if [ -n "$2" ]; then
+    set -u
+    repository_owner=$2
+    remote=$3
+    set +u
+    [ $# -eq 3 ]
+fi
+
+[ -n "$version" ]
+
+for tool in \
+    bundle \
+    gh \
+    ruby \
+    timeout \
+; do
+    if ! command -v "$tool" >/dev/null; then
+        echo >&2 "$0: missing command: $1"
+        exit 1
+    fi
+done
+
+repo_root="$(dirname "$0")/.."
+fish_site=$repo_root/../fish-site
+
+for path in . "$fish_site"
+do
+    if ! git -C "$path" diff HEAD --quiet; then
+        echo >&2 "$0: index and worktree must be clean"
+        exit 1
+    fi
+done
+
+if git tag | grep -qxF "$version"; then
+    echo >&2 "$0: tag $version already exists"
+    exit 1
+fi
+
+sed -n 1p CHANGELOG.rst | grep -q '^fish .*(released ???)$'
+sed -n 2p CHANGELOG.rst | grep -q '^===*$'
+
+changelog_title="fish $version (released $(date +'%B %d, %Y'))"
+sed -i \
+    -e "1c$changelog_title" \
+    -e "2c$(printf %s "$changelog_title" | sed s/./=/g)" \
+    CHANGELOG.rst
+
+CommitVersion() {
+    sed -i "s/^version = \".*\"/version = \"$1\"/g" Cargo.toml
+    cargo fetch --offline
+    git add CHANGELOG.rst Cargo.toml Cargo.lock
+    git commit -m "$2
+
+Created by ./build_tools/release.sh $version"
+}
+
+CommitVersion "$version" "Release $version"
+
+# N.B. this is not GPG-signed.
+git tag --annotate --message="Release $version" $version
+
+git push $remote $version
+
+gh() {
+    command gh --repo "$repository_owner/fish-shell" "$@"
+}
+
+run_id=
+while [ -z "$run_id" ] && sleep 5
+do
+    run_id=$(gh run list \
+                --json=databaseId --jq=.[].databaseId \
+                --workflow=release.yml --limit=1 \
+                --commit="$(git rev-parse "$version^{commit}")")
+done
+
+# Update fishshell.com
+tag_oid=$(git rev-parse "$version")
+tmpdir=$(mktemp -d)
+while ! \
+    gh release download "$version" --dir="$tmpdir" \
+        --pattern="fish-$version.tar.xz"
+do
+    timeout 30 gh run watch "$run_id" ||:
+done
+actual_tag_oid=$(git ls-remote "$remote" |
+    awk '$2 == "refs/tags/'"$version"'" { print $1 }')
+[ "$tag_oid" = "$actual_tag_oid" ]
+tar -C "$tmpdir" xf fish-$version.tar.xz
+minor_version=${version%.*}
+CopyDocs() {
+    rm -rf "fish-site/site/docs/$1"
+    cp -r "$tmpdir/fish-$version/user_doc/html" "fish-site/site/docs/$1"
+    git -C fish-site add "site/docs/$1"
+}
+CopyDocs "$minor_version"
+latest_release=$(
+    releases=$(git tag | grep '^[0-9]*\.[0-9]*\.[0-9]*.*' |
+        sed $(: "De-prioritize release candidates (1.2.3-rc0)") \
+        's/-/~/g' | LC_ALL=C sort --version-sort)
+    printf %s\\n "$releases" | tail -1
+)
+if [ "$version" = "$latest_release" ]; then
+    CopyDocs current
+fi
+rm -rf "$tmpdir"
+(
+    cd "$fish_site"
+    make new-release
+    git add -u
+    git commit --message="$(printf %s "\
+        | Release $version
+        |
+        | Created by ../fish-shell/build_tools/release.sh
+    " | sed 's,^\s*| ,,')"
+)
+
+# N.B. --exit-status doesn't fail reliably.
+gh run view "$run_id" --verbose --log-failed --exit-status
+
+while {
+    ! draft=$(gh release view "$version" --json=isDraft --jq=.isDraft) \
+    || [ "$draft" = true ]
+}
+do
+    sleep 20
+done
+
+(
+    cd "$fish_site"
+    git push git@github.com:$repository_owner/fish-site
+)
+
+if git merge-base --is-ancestor $remote/master $version
+then
+    git push $remote $version:master
+else
+    # Probably on an integration branch.
+    # TODO Maybe push when that's safe (or move this to CI).
+    :
+fi
+
+if [ "$repository_owner" = fish-shell ]; then {
+    mail=$(mktemp)
+    cat >$mail <<EOF
+From: $(git var GIT_AUTHOR_IDENT | sed 's/ [0-9]* +[0-9]*$//')
+To: fish-users Mailing List <fish-users@lists.sourceforge.net>
+Subject: fish $version released
+
+See https://github.com/fish-shell/fish-shell/releases/tag/$version
+EOF
+    git send-email --suppress-cc=all $mail
+    rm $mail
+} fi
+
+changelog=$(cat - CHANGELOG.rst <<EOF
+fish ?.?.? (released ???)
+=========================
+
+EOF
+)
+printf %s\\n "$changelog" >CHANGELOG.rst
+CommitVersion ${version}-snapshot "start new cycle"
+
+exit
+
+}

cmake/MacApp.cmake

@@ -24,7 +24,7 @@ add_executable(fish_macapp EXCLUDE_FROM_ALL
 
 # Compute the version. Note this is done at generation time, not build time,
 # so cmake must be re-run after version changes for the app to be updated. But
-# generally this will be run by make_pkg.sh which always re-runs cmake.
+# generally this will be run by make_macos_pkg.sh which always re-runs cmake.
 execute_process(
     COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build_tools/git_version_gen.sh --stdout
     COMMAND cut -d- -f1
@@ -32,7 +32,7 @@ execute_process(
     OUTPUT_STRIP_TRAILING_WHITESPACE)
 
 
-# Note CMake appends .app, so the real output name will be fish.app. 
+# Note CMake appends .app, so the real output name will be fish.app.
 # This target does not include the 'base' resource.
 set_target_properties(fish_macapp PROPERTIES OUTPUT_NAME "fish")
 

doc_src/conf.py

@@ -45,8 +45,8 @@ def issue_role(name, rawtext, text, lineno, inliner, options=None, content=None)
     return [link], []
 
 
-def do_not_use_fish_indent_for_man(app):
-    if app.builder.name == "man":
+def remove_fish_indent_lexer(app):
+    if app.builder.name in ("man", "markdown"):
         del lexers["fish-docs-samples"]
 
 
@@ -65,7 +65,7 @@ def setup(app):
     app.add_config_value("issue_url", default=None, rebuild="html")
     app.add_role("issue", issue_role)
 
-    app.connect("builder-inited", do_not_use_fish_indent_for_man)
+    app.connect("builder-inited", remove_fish_indent_lexer)
 
 
 # The default language to assume