surrealpatch/.github/workflows/reusable_publish_version.yml

on:
  workflow_call:
    inputs:
      environment:
        required: true
        type: string
        description: "The name of this release environment. It can be a 'nightly', 'beta' or 'release'"
      git-ref:
        required: true
        type: string
        description: "The git ref of this release version. All 'actions/checkout' steps will use it"
      bump-version:
        required: false
        type: boolean
        default: false
        description: "Bump the version of the current beta if this is not the initial one"
      latest:
        required: false
        type: boolean
        default: false
        description: "Consider this release as the latest one and update the Docker image tag and the binary pointer for the installers"
      publish:
        required: false
        type: boolean
        default: false
        description: "Whether to publish this release"
      create-release:
        required: false
        type: boolean
        default: false
        description: "Create a GitHub release"
      http-compression:
        required: false
        type: boolean
        default: true
        description: "Enable HTTP compression in binaries"
      ml:
        required: false
        type: boolean
        default: true
        description: "Enable ML support in binaries"

defaults:
  run:
    shell: bash

jobs:
  prepare-vars:
    name: Prepare vars
    runs-on: ubuntu-latest
    outputs:
      git-ref: ${{ steps.outputs.outputs.git-ref }}
      name: ${{ steps.outputs.outputs.name }}
      build-metadata: ${{ steps.outputs.outputs.build-metadata }}
    steps:
      - name: Install stable toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: 1.74.1

      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.git-ref }}

      - name: Install a TOML parser
        run: cargo install --force --locked --version 0.8.1 taplo-cli

      - name: Configure git
        run: |
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config user.name "github-actions[bot]"
          git config --add --bool push.autoSetupRemote true

      - name: Patch release version
        if: ${{ inputs.environment == 'stable' }}
        run: |
          set -x

          currentVersion=$(/home/runner/.cargo/bin/taplo get -f lib/Cargo.toml "package.version")

          if [[ $currentVersion == *"-beta"* ]]; then
            git push origin --delete releases/stable || true
            git checkout -b releases/stable
            major=$(echo $currentVersion | tr "." "\n" | sed -n 1p)
            minor=$(echo $currentVersion | tr "." "\n" | sed -n 2p)
            version=${major}.${minor}.0

            # Bump the crate version
            sed -i "s#^version = \".*\"#version = \"${version}\"#" Cargo.toml
            sed -i "s#^version = \".*\"#version = \"${version}\"#" lib/Cargo.toml

            # Update Cargo.lock without updating dependency versions
            cargo check --no-default-features --features storage-mem

            # Commit changes
            git commit -am "Prepare v${version} release"
          else
            version=${currentVersion}
          fi

          # Create the tag
          git tag -a v${version} -m "Release ${version}" || true

      - name: Create or patch beta branch
        if: ${{ inputs.environment == 'beta' }}
        run: |
          set -x

          currentVersion=$(/home/runner/.cargo/bin/taplo get -f lib/Cargo.toml "package.version")

          if [[ $currentVersion == *"-beta"* ]]; then
            if [[ "${{ inputs.bump-version }}" == "true" ]]; then
              major=$(echo $currentVersion | tr "." "\n" | sed -n 1p)
              minor=$(echo $currentVersion | tr "." "\n" | sed -n 2p)
              patchAndMeta=$(echo $currentVersion | tr "." "\n" | sed -n 3p)
              betaNum=$(echo $currentVersion | tr "." "\n" | sed -n 4p)
              betaVersion=${major}.${minor}.${patchAndMeta}.$(($betaNum + 1))
            else
              betaVersion=$currentVersion
            fi
          else
            git checkout -b releases/beta
            major=$(echo $currentVersion | tr "." "\n" | sed -n 1p)
            minor=$(echo $currentVersion | tr "." "\n" | sed -n 2p)
            betaVersion=${major}.$(($minor + 1)).0-beta.1
          fi

          # Bump the crate version
          sed -i "s#^version = \".*\"#version = \"${betaVersion}\"#" Cargo.toml
          sed -i "s#^version = \".*\"#version = \"${betaVersion}\"#" lib/Cargo.toml

          # Update Cargo.lock without updating dependency versions
          cargo check --no-default-features --features storage-mem

          # Commit changes
          git commit -am "Prepare v${betaVersion} release" || true

          # Create the tag
          git tag -a v${betaVersion} -m "Release ${betaVersion}" || true

      - name: Push changes
        if: ${{ inputs.publish && (inputs.environment == 'beta' || inputs.environment == 'stable') }}
        run: git push

      - name: Push tag
        if: ${{ inputs.publish && (inputs.environment == 'beta' || inputs.environment == 'stable') }}
        run: git push --tags || true

      - name: Set outputs
        id: outputs
        run: |
          set -x

          version=$(/home/runner/.cargo/bin/taplo get -f lib/Cargo.toml "package.version")

          if [[ "${{ inputs.publish }}" == "true" && ("${{ inputs.environment }}" == "beta" || "${{ inputs.environment }}" == "stable")  ]]; then
            echo "git-ref=v${version}" >> $GITHUB_OUTPUT
          else
            echo "git-ref=${{ inputs.git-ref }}" >> $GITHUB_OUTPUT
          fi

          if [[ "${{ inputs.environment }}" == "nightly" ]]; then
            echo "name=${{ inputs.environment }}" >> $GITHUB_OUTPUT

            date=$(git show --no-patch --format=%ad --date=format:%Y%m%d)
            rev=$(git rev-parse --short HEAD)
            echo "build-metadata=${date}.${rev}" >> $GITHUB_OUTPUT
          else
            echo "name=v${version}" >> $GITHUB_OUTPUT
            echo "build-metadata=" >> $GITHUB_OUTPUT
          fi

  test:
    name: Test
    needs: [prepare-vars]
    runs-on: ubuntu-latest-16-cores
    steps:
      - name: Install stable toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: 1.75.0

      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.prepare-vars.outputs.git-ref }}

      - name: Setup cache
        uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ needs.prepare-vars.outputs.git-ref == 'main' }}

      - name: Install cargo-llvm-cov
        uses: taiki-e/install-action@cargo-llvm-cov

      - name: Install cargo-make
        run: cargo install --debug --locked cargo-make

      - name: Test workspace + coverage
        run: cargo make ci-workspace-coverage

      - name: Debug info
        if: always()
        run: |
          set -x
          free -m
          df -h
          ps auxf
          cat /tmp/surrealdb.log || true

      - name: Upload coverage report
        uses: actions/upload-artifact@v3
        with:
          name: code-coverage-report
          path: target/llvm-cov/html/
          retention-days: 5

  lint:
    name: Lint
    needs: [prepare-vars]
    runs-on: ubuntu-latest
    steps:

      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.prepare-vars.outputs.git-ref }}

      - name: Install stable toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: 1.75.0
          targets: wasm32-unknown-unknown
          components: rustfmt, clippy

      - name: Install cargo-make
        run: cargo install --debug --locked cargo-make

      - name: Check workspace
        run: cargo make ci-check

      - name: Check format
        run: cargo make ci-format

      - name: Check wasm
        run: cargo make ci-check-wasm

      - name: Check clippy
        run: cargo make ci-clippy

  docker-build:
    name: Build Docker images
    needs: [prepare-vars]
    uses: ./.github/workflows/reusable_docker.yml
    with:
      git-ref: ${{ needs.prepare-vars.outputs.git-ref }}
      tag-prefix: ${{ needs.prepare-vars.outputs.name }}
      build: true
      push: false
    secrets: inherit

  build:
    name: Build ${{ matrix.arch }} binary
    needs: [prepare-vars]
    strategy:
      fail-fast: false
      matrix:
        include:
          # MacOS amd64
          - arch: x86_64-apple-darwin
            runner: macos-latest-xl
            file: surreal-${{ needs.prepare-vars.outputs.name }}.darwin-amd64
            build-step: |
              set -x

              # Prepare deps
              brew install protobuf

              # Build
              features=storage-tikv
              if [[ "${{ inputs.http-compression }}" == "true" ]]; then
                features=${features},http-compression
              fi
              if [[ "${{ inputs.ml }}" == "true" ]]; then
                features=${features},ml
              fi
              cargo build --features $features --release --locked --target x86_64-apple-darwin

              # Package
              cp target/x86_64-apple-darwin/release/surreal surreal
              chmod +x surreal
              ./surreal version
              tar -zcvf surreal-${{ needs.prepare-vars.outputs.name }}.darwin-amd64.tgz surreal
              echo $(shasum -a 256 surreal-${{ needs.prepare-vars.outputs.name }}.darwin-amd64.tgz | cut -f1 -d' ') > surreal-${{ needs.prepare-vars.outputs.name }}.darwin-amd64.txt

          # MacOS arm64
          - arch: aarch64-apple-darwin
            runner: macos-latest-xl
            file: surreal-${{ needs.prepare-vars.outputs.name }}.darwin-arm64
            build-step: |
              set -x

              # Prepare deps
              brew install protobuf

              # Build
              features=storage-tikv
              if [[ "${{ inputs.http-compression }}" == "true" ]]; then
                features=${features},http-compression
              fi
              if [[ "${{ inputs.ml }}" == "true" ]]; then
                features=${features},ml
              fi
              cargo build --features $features --release --locked --target aarch64-apple-darwin

              # Package
              cp target/aarch64-apple-darwin/release/surreal surreal
              chmod +x surreal
              tar -zcvf surreal-${{ needs.prepare-vars.outputs.name }}.darwin-arm64.tgz surreal
              echo $(shasum -a 256 surreal-${{ needs.prepare-vars.outputs.name }}.darwin-arm64.tgz | cut -f1 -d' ') > surreal-${{ needs.prepare-vars.outputs.name }}.darwin-arm64.txt

          # Linux amd64
          - arch: x86_64-unknown-linux-gnu
            runner: ["self-hosted", "amd64", "builder"]
            file: surreal-${{ needs.prepare-vars.outputs.name }}.linux-amd64
            build-step: |
              set -x

              # Build
              features=storage-tikv
              if [[ "${{ inputs.http-compression }}" == "true" ]]; then
                features=${features},http-compression
              fi
              if [[ "${{ inputs.ml }}" == "true" ]]; then
                features=${features},ml
              fi
              docker build \
                --platform linux/amd64 \
                --build-arg="CARGO_EXTRA_FEATURES=${features}" \
                --build-arg="SURREAL_BUILD_METADATA=${SURREAL_BUILD_METADATA}" \
                -t binary \
                -f docker/Dockerfile.binary \
                .
              docker create --name binary binary
              docker cp binary:/surrealdb/target/release/surreal surreal
              
              # Package
              chmod +x surreal
              ./surreal version
              tar -zcvf surreal-${{ needs.prepare-vars.outputs.name }}.linux-amd64.tgz surreal
              echo $(shasum -a 256 surreal-${{ needs.prepare-vars.outputs.name }}.linux-amd64.tgz | cut -f1 -d' ') > surreal-${{ needs.prepare-vars.outputs.name }}.linux-amd64.txt

          # Linux arm64
          - arch: aarch64-unknown-linux-gnu
            runner: ["self-hosted", "arm64", "builder"]
            file: surreal-${{ needs.prepare-vars.outputs.name }}.linux-arm64
            build-step: |
              set -x

              # Build
              features=storage-tikv
              if [[ "${{ inputs.http-compression }}" == "true" ]]; then
                features=${features},http-compression
              fi
              if [[ "${{ inputs.ml }}" == "true" ]]; then
                features=${features},ml
              fi
              docker build \
                --platform linux/arm64 \
                --build-arg="CARGO_EXTRA_FEATURES=${features}" \
                --build-arg="SURREAL_BUILD_METADATA=${SURREAL_BUILD_METADATA}" \
                -t binary \
                -f docker/Dockerfile.binary \
                .
              docker create --name binary binary
              docker cp binary:/surrealdb/target/release/surreal surreal

              # Package
              chmod +x surreal
              ./surreal version
              tar -zcvf surreal-${{ needs.prepare-vars.outputs.name }}.linux-arm64.tgz surreal
              echo $(shasum -a 256 surreal-${{ needs.prepare-vars.outputs.name }}.linux-arm64.tgz | cut -f1 -d' ') > surreal-${{ needs.prepare-vars.outputs.name }}.linux-arm64.txt

          # Windows amd64
          - arch: x86_64-pc-windows-msvc
            runner: windows-latest
            file: surreal-${{ needs.prepare-vars.outputs.name }}.windows-amd64
            build-step: |
              set -x

              # Prepare deps
              vcpkg integrate install

              # Build
              features=storage-tikv
              if [[ "${{ inputs.http-compression }}" == "true" ]]; then
                features=${features},http-compression
              fi
              if [[ "${{ inputs.ml }}" == "true" ]]; then
                features=${features},ml
              fi
              cargo build --features $features --release --locked --target x86_64-pc-windows-msvc

              # Package
              ./target/x86_64-pc-windows-msvc/release/surreal.exe version
              cp target/x86_64-pc-windows-msvc/release/surreal.exe surreal-${{ needs.prepare-vars.outputs.name }}.windows-amd64.exe
              echo $(shasum -a 256 surreal-${{ needs.prepare-vars.outputs.name }}.windows-amd64.exe | cut -f1 -d' ') > surreal-${{ needs.prepare-vars.outputs.name }}.windows-amd64.txt

    runs-on: ${{ matrix.runner }}
    steps:
      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.prepare-vars.outputs.git-ref }}

      - name: Checkout docker
        uses: actions/checkout@v4
        with:
          path: _docker
  
      # Replace docker files. It allows us to test new Dockerfiles with workflow_dispatch and a custom git ref.
      # When triggered by a push or a schedule, this git ref will be the same as 'inputs.git-ref'
      - name: Replace docker files
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          rm -rf docker .dockerignore
          mv _docker/docker .
          mv _docker/.dockerignore .
          rm -rf _docker

      - name: Install stable toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: 1.74.1
          targets: ${{ matrix.arch }}

      - name: Output package versions
        run: |
          set -x
          set +e
          go version ; cargo version ; rustc --version ; cmake --version ; gcc --version ; g++ --version ; perl -v

      - name: Build step
        env:
          SURREAL_BUILD_METADATA: ${{ needs.prepare-vars.outputs.build-metadata }}
        run: ${{ matrix.build-step }}

      - name: Upload artifacts
        uses: actions/upload-artifact@v3
        with:
          name: ${{ matrix.file }}
          path: |
            surreal
            ${{ matrix.file }}.tgz
            ${{ matrix.file }}.txt
            ${{ matrix.file }}.exe

  publish:
    name: Publish crate and artifacts binaries
    needs: [prepare-vars, test, lint, build, docker-build]
    if: ${{ inputs.publish }}
    environment: ${{ inputs.environment }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout sources
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.prepare-vars.outputs.git-ref }}

      - name: Install stable toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: 1.74.1

      - name: Install release-plz
        run: cargo install --force --locked --version 0.3.30 release-plz

      - name: Install a TOML parser
        if: ${{ inputs.environment == 'beta' }}
        run: cargo install --force --locked --version 0.8.1 taplo-cli

      - name: Create a temporary branch
        run: git checkout -b crate

      - name: Configure release-plz
        run: |
          cat << EOF > /tmp/release-plz.toml
          [workspace]
          changelog_update = false
          git_release_enable = false
          semver_check = false
          git_tag_enable = false
          EOF

      - name: Patch beta crate version
        if: ${{ inputs.environment == 'beta' }}
        run: |
          set -x

          # Derive crate version
          currentVersion=$(/home/runner/.cargo/bin/taplo get -f lib/Cargo.toml "package.version")
          major=$(echo $currentVersion | tr "." "\n" | sed -n 1p)
          minor=$(echo $currentVersion | tr "." "\n" | sed -n 2p)
          betaNum=$(echo $currentVersion | tr "." "\n" | sed -n 4p)
          version=${major}.${minor}.$(($betaNum - 1))

          # Update crate version
          sed -i "s#^version = \".*\"#version = \"${version}\"#" lib/Cargo.toml

      - name:  Patch nightly crate version
        if: ${{ inputs.environment == 'nightly' }}
        run: |
          # Get the date and time of the last commit
          date=$(git show --no-patch --format=%ad --date=format:%Y%m%d)
          time=$(git show --no-patch --format=%ad --date=format:%H%M%S)

          # Update the version to a nightly one
          # This sets the nightly version to something like `1.20231117.1130416`
          # The 1 at the beginning of the patch number is just so it never starts with zero
          sed -i "s#^version = \"\([[:digit:]]*\)\..*\"#version = \"\1.${date}.1${time}\"#" lib/Cargo.toml

      - name: Patch crate name and description
        if: ${{ inputs.environment == 'nightly' || inputs.environment == 'beta' }}
        run: |
          set -x

          # Patch crate name
          sed -i "0,/surrealdb/s//surrealdb-${{ inputs.environment }}/" lib/Cargo.toml

          # Patch the description
          sed -i "s#^description = \".*\"#description = \"A ${{ inputs.environment }} release of the surrealdb crate\"#" lib/Cargo.toml

          # Temporarily commit patches
          # These should not be pushed back to the repo
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config user.name "github-actions[bot]"
          git commit -am "Name and version patches"

      - name: Publish the crate
        env:
          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
        run: /home/runner/.cargo/bin/release-plz release --config /tmp/release-plz.toml

      - name: Download artifacts
        uses: actions/download-artifact@v3
        with:
          path: artifacts

      - name: Publish release
        uses: softprops/action-gh-release@v1
        if: ${{ inputs.create-release }}
        with:
          name: "Release ${{ needs.prepare-vars.outputs.git-ref }}"
          tag_name: ${{ needs.prepare-vars.outputs.git-ref }}
          prerelease: ${{ inputs.environment == 'beta' || inputs.environment == 'nightly' }}
          fail_on_unmatched_files: true
          files: |
            LICENSE
            artifacts/surreal-${{ needs.prepare-vars.outputs.git-ref }}.*/*.tgz
            artifacts/surreal-${{ needs.prepare-vars.outputs.git-ref }}.*/*.exe

      - name: Configure AWS
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-east-2
          aws-access-key-id: ${{ secrets.AMAZON_ACCESS_KEY }}
          aws-secret-access-key: ${{ secrets.AMAZON_SECRET_KEY }}

      - name: Set latest release version
        if: ${{ inputs.create-release && inputs.latest }}
        run: |
          echo ${{ needs.prepare-vars.outputs.git-ref }} > latest.txt
          aws s3 cp --cache-control 'no-store' latest.txt s3://download.surrealdb.com/latest.txt

      - name: Set latest beta version
        if: ${{ inputs.publish && inputs.environment == 'beta' }}
        run: |
          echo ${{ needs.prepare-vars.outputs.git-ref }} > beta.txt
          aws s3 cp --cache-control 'no-store' beta.txt s3://download.surrealdb.com/beta.txt

      - name: Publish binaries
        run: |
          for file in artifacts/**/*.{tgz,txt,exe}; do
            aws s3 cp --cache-control 'no-store' $file s3://download.surrealdb.com/${{ needs.prepare-vars.outputs.name }}/
          done

  docker-publish:
    name: Publish Docker images
    needs: [prepare-vars, publish]
    uses: ./.github/workflows/reusable_docker.yml
    with:
      git-ref: ${{ needs.prepare-vars.outputs.git-ref }}
      tag-prefix: ${{ needs.prepare-vars.outputs.name }}
      latest: ${{ inputs.latest }}
      build: false
      push: true
    secrets: inherit

  package-macos:
    name: Package and publish macOS universal binary
    needs: [prepare-vars, publish]
    runs-on: macos-latest
    env:
      FILE: surreal-${{ needs.prepare-vars.outputs.name }}.darwin-universal
    steps:
      - name: Download amd64 binary
        uses: actions/download-artifact@v3
        with:
          name: surreal-${{ needs.prepare-vars.outputs.name }}.darwin-amd64
          path: amd64

      - name: Download arm64 binary
        uses: actions/download-artifact@v3
        with:
          name: surreal-${{ needs.prepare-vars.outputs.name }}.darwin-arm64
          path: arm64

      - name: Configure AWS
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-east-2
          aws-access-key-id: ${{ secrets.AMAZON_ACCESS_KEY }}
          aws-secret-access-key: ${{ secrets.AMAZON_SECRET_KEY }}

      - name: Package universal MacOS binary
        run: |
          lipo -create -output surreal amd64/surreal arm64/surreal
          chmod +x surreal
          tar -zcvf $FILE.tgz surreal
          echo $(shasum -a 256 $FILE.tgz | cut -f1 -d' ') > $FILE.txt

      - name: Publish universal MacOS binary
        if: ${{ inputs.publish }}
        run: |
          aws s3 cp --cache-control 'no-store' $FILE.tgz s3://download.surrealdb.com/${{ needs.prepare-vars.outputs.name }}/
          aws s3 cp --cache-control 'no-store' $FILE.txt s3://download.surrealdb.com/${{ needs.prepare-vars.outputs.name }}/

  propagate:
    name: Propagate binaries to all regions
    if: ${{ inputs.publish }}
    needs: [publish, package-macos]
    runs-on: ubuntu-latest
    steps:
      - name: Configure AWS
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-region: us-east-2
          aws-access-key-id: ${{ secrets.AMAZON_ACCESS_KEY }}
          aws-secret-access-key: ${{ secrets.AMAZON_SECRET_KEY }}

      - name: Distribute binaries
        run: |
          regions=("af-south-1" "ap-east-1" "ap-south-1" "ap-southeast-1" "ap-southeast-2" "ca-central-1" "eu-central-1" "eu-west-2" "me-south-1" "sa-east-1" "us-west-2")
          for region in ${regions[@]}; do
              aws s3 sync --delete --storage-class INTELLIGENT_TIERING --source-region eu-west-2 --region ${region} s3://download.surrealdb.com s3://download.${region}.surrealdb.com
          done