surrealpatch/.github/workflows/reusable_publish_version.yml
2024-01-05 19:07:17 +00:00

659 lines
24 KiB
YAML

on:
workflow_call:
inputs:
environment:
required: true
type: string
description: "The name of this release environment. It can be a 'nightly', 'beta' or 'release'"
git-ref:
required: true
type: string
description: "The git ref of this release version. All 'actions/checkout' steps will use it"
bump-version:
required: false
type: boolean
default: false
description: "Bump the version of the current beta if this is not the initial one"
latest:
required: false
type: boolean
default: false
description: "Consider this release as the latest one and update the Docker image tag and the binary pointer for the installers"
publish:
required: false
type: boolean
default: false
description: "Whether to publish this release"
create-release:
required: false
type: boolean
default: false
description: "Create a GitHub release"
http-compression:
required: false
type: boolean
default: true
description: "Enable HTTP compression in binaries"
ml:
required: false
type: boolean
default: true
description: "Enable ML support in binaries"
defaults:
run:
shell: bash
jobs:
prepare-vars:
name: Prepare vars
runs-on: ubuntu-latest
outputs:
git-ref: ${{ steps.outputs.outputs.git-ref }}
name: ${{ steps.outputs.outputs.name }}
build-metadata: ${{ steps.outputs.outputs.build-metadata }}
steps:
- name: Install stable toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: 1.74.1
- name: Checkout sources
uses: actions/checkout@v4
with:
ref: ${{ inputs.git-ref }}
- name: Install a TOML parser
run: cargo install --force --locked --version 0.8.1 taplo-cli
- name: Configure git
run: |
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config user.name "github-actions[bot]"
git config --add --bool push.autoSetupRemote true
- name: Patch release version
if: ${{ inputs.environment == 'stable' }}
run: |
set -x
currentVersion=$(/home/runner/.cargo/bin/taplo get -f lib/Cargo.toml "package.version")
if [[ $currentVersion == *"-beta"* ]]; then
git push origin --delete releases/stable || true
git checkout -b releases/stable
major=$(echo $currentVersion | tr "." "\n" | sed -n 1p)
minor=$(echo $currentVersion | tr "." "\n" | sed -n 2p)
betaNum=$(echo $currentVersion | tr "." "\n" | sed -n 4p)
version=${major}.${minor}.$(($betaNum - 1))
# Bump the crate version
sed -i "s#^version = \".*\"#version = \"${version}\"#" Cargo.toml
sed -i "s#^version = \".*\"#version = \"${version}\"#" lib/Cargo.toml
# Update Cargo.lock without updating dependency versions
cargo check --no-default-features --features storage-mem
# Commit changes
git commit -am "Prepare v${version} release"
else
version=${currentVersion}
fi
# Create the tag
git tag -a v${version} -m "Release ${version}" || true
- name: Create or patch beta branch
if: ${{ inputs.environment == 'beta' }}
run: |
set -x
currentVersion=$(/home/runner/.cargo/bin/taplo get -f lib/Cargo.toml "package.version")
if [[ $currentVersion == *"-beta"* ]]; then
if [[ "${{ inputs.bump-version }}" == "true" ]]; then
major=$(echo $currentVersion | tr "." "\n" | sed -n 1p)
minor=$(echo $currentVersion | tr "." "\n" | sed -n 2p)
patchAndMeta=$(echo $currentVersion | tr "." "\n" | sed -n 3p)
betaNum=$(echo $currentVersion | tr "." "\n" | sed -n 4p)
betaVersion=${major}.${minor}.${patchAndMeta}.$(($betaNum + 1))
else
betaVersion=$currentVersion
fi
else
git checkout -b releases/beta
major=$(echo $currentVersion | tr "." "\n" | sed -n 1p)
minor=$(echo $currentVersion | tr "." "\n" | sed -n 2p)
betaVersion=${major}.$(($minor + 1)).0-beta.1
fi
# Bump the crate version
sed -i "s#^version = \".*\"#version = \"${betaVersion}\"#" Cargo.toml
sed -i "s#^version = \".*\"#version = \"${betaVersion}\"#" lib/Cargo.toml
# Update Cargo.lock without updating dependency versions
cargo check --no-default-features --features storage-mem
# Commit changes
git commit -am "Prepare v${betaVersion} release" || true
# Create the tag
git tag -a v${betaVersion} -m "Release ${betaVersion}" || true
- name: Push changes
if: ${{ inputs.publish && (inputs.environment == 'beta' || inputs.environment == 'stable') }}
run: git push
- name: Push tag
if: ${{ inputs.publish && (inputs.environment == 'beta' || inputs.environment == 'stable') }}
run: git push --tags || true
- name: Set outputs
id: outputs
run: |
set -x
version=$(/home/runner/.cargo/bin/taplo get -f lib/Cargo.toml "package.version")
if [[ "${{ inputs.publish }}" == "true" && ("${{ inputs.environment }}" == "beta" || "${{ inputs.environment }}" == "stable") ]]; then
echo "git-ref=v${version}" >> $GITHUB_OUTPUT
else
echo "git-ref=${{ inputs.git-ref }}" >> $GITHUB_OUTPUT
fi
if [[ "${{ inputs.environment }}" == "nightly" ]]; then
echo "name=${{ inputs.environment }}" >> $GITHUB_OUTPUT
date=$(git show --no-patch --format=%ad --date=format:%Y%m%d)
rev=$(git rev-parse --short HEAD)
echo "build-metadata=${date}.${rev}" >> $GITHUB_OUTPUT
else
echo "name=v${version}" >> $GITHUB_OUTPUT
echo "build-metadata=" >> $GITHUB_OUTPUT
fi
test:
name: Test
needs: [prepare-vars]
runs-on: ubuntu-latest-16-cores
steps:
- name: Install stable toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: 1.71.1
- name: Checkout sources
uses: actions/checkout@v4
with:
ref: ${{ needs.prepare-vars.outputs.git-ref }}
- name: Setup cache
uses: Swatinem/rust-cache@v2
with:
save-if: ${{ needs.prepare-vars.outputs.git-ref == 'main' }}
- name: Install cargo-llvm-cov
uses: taiki-e/install-action@cargo-llvm-cov
- name: Install cargo-make
run: cargo install --debug --locked cargo-make
- name: Test workspace + coverage
run: cargo make ci-workspace-coverage
- name: Debug info
if: always()
run: |
set -x
free -m
df -h
ps auxf
cat /tmp/surrealdb.log || true
- name: Upload coverage report
uses: actions/upload-artifact@v3
with:
name: code-coverage-report
path: target/llvm-cov/html/
retention-days: 5
lint:
name: Lint
needs: [prepare-vars]
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v4
with:
ref: ${{ needs.prepare-vars.outputs.git-ref }}
- name: Install stable toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: 1.71.1
targets: wasm32-unknown-unknown
components: rustfmt, clippy
- name: Install cargo-make
run: cargo install --debug --locked cargo-make
- name: Check workspace
run: cargo make ci-check
- name: Check format
run: cargo make ci-format
- name: Check wasm
run: cargo make ci-check-wasm
- name: Check clippy
run: cargo make ci-clippy
docker-build:
name: Build Docker images
needs: [prepare-vars]
uses: ./.github/workflows/reusable_docker.yml
with:
git-ref: ${{ needs.prepare-vars.outputs.git-ref }}
tag-prefix: ${{ needs.prepare-vars.outputs.name }}
build: true
push: false
secrets: inherit
build:
name: Build ${{ matrix.arch }} binary
needs: [prepare-vars]
strategy:
fail-fast: false
matrix:
include:
# MacOS amd64
- arch: x86_64-apple-darwin
runner: macos-latest-xl
file: surreal-${{ needs.prepare-vars.outputs.name }}.darwin-amd64
build-step: |
set -x
# Prepare deps
brew install protobuf
# Build
features=storage-tikv
if [[ "${{ inputs.http-compression }}" == "true" ]]; then
features=${features},http-compression
fi
if [[ "${{ inputs.ml }}" == "true" ]]; then
features=${features},ml
fi
cargo build --features $features --release --locked --target x86_64-apple-darwin
# Package
cp target/x86_64-apple-darwin/release/surreal surreal
chmod +x surreal
./surreal version
tar -zcvf surreal-${{ needs.prepare-vars.outputs.name }}.darwin-amd64.tgz surreal
echo $(shasum -a 256 surreal-${{ needs.prepare-vars.outputs.name }}.darwin-amd64.tgz | cut -f1 -d' ') > surreal-${{ needs.prepare-vars.outputs.name }}.darwin-amd64.txt
# MacOS arm64
- arch: aarch64-apple-darwin
runner: macos-latest-xl
file: surreal-${{ needs.prepare-vars.outputs.name }}.darwin-arm64
build-step: |
set -x
# Prepare deps
brew install protobuf
# Build
features=storage-tikv
if [[ "${{ inputs.http-compression }}" == "true" ]]; then
features=${features},http-compression
fi
if [[ "${{ inputs.ml }}" == "true" ]]; then
features=${features},ml
fi
cargo build --features $features --release --locked --target aarch64-apple-darwin
# Package
cp target/aarch64-apple-darwin/release/surreal surreal
chmod +x surreal
tar -zcvf surreal-${{ needs.prepare-vars.outputs.name }}.darwin-arm64.tgz surreal
echo $(shasum -a 256 surreal-${{ needs.prepare-vars.outputs.name }}.darwin-arm64.tgz | cut -f1 -d' ') > surreal-${{ needs.prepare-vars.outputs.name }}.darwin-arm64.txt
# Linux amd64
- arch: x86_64-unknown-linux-gnu
runner: ["self-hosted", "amd64", "builder"]
file: surreal-${{ needs.prepare-vars.outputs.name }}.linux-amd64
build-step: |
set -x
# Build
features=storage-tikv
if [[ "${{ inputs.http-compression }}" == "true" ]]; then
features=${features},http-compression
fi
if [[ "${{ inputs.ml }}" == "true" ]]; then
features=${features},ml
fi
docker build \
--platform linux/amd64 \
--build-arg="CARGO_EXTRA_FEATURES=${features}" \
--build-arg="SURREAL_BUILD_METADATA=${SURREAL_BUILD_METADATA}" \
-t binary \
-f docker/Dockerfile.binary \
.
docker create --name binary binary
docker cp binary:/surrealdb/target/release/surreal surreal
# Package
chmod +x surreal
./surreal version
tar -zcvf surreal-${{ needs.prepare-vars.outputs.name }}.linux-amd64.tgz surreal
echo $(shasum -a 256 surreal-${{ needs.prepare-vars.outputs.name }}.linux-amd64.tgz | cut -f1 -d' ') > surreal-${{ needs.prepare-vars.outputs.name }}.linux-amd64.txt
# Linux arm64
- arch: aarch64-unknown-linux-gnu
runner: ["self-hosted", "arm64", "builder"]
file: surreal-${{ needs.prepare-vars.outputs.name }}.linux-arm64
build-step: |
set -x
# Build
features=storage-tikv
if [[ "${{ inputs.http-compression }}" == "true" ]]; then
features=${features},http-compression
fi
if [[ "${{ inputs.ml }}" == "true" ]]; then
features=${features},ml
fi
docker build \
--platform linux/arm64 \
--build-arg="CARGO_EXTRA_FEATURES=${features}" \
--build-arg="SURREAL_BUILD_METADATA=${SURREAL_BUILD_METADATA}" \
-t binary \
-f docker/Dockerfile.binary \
.
docker create --name binary binary
docker cp binary:/surrealdb/target/release/surreal surreal
# Package
chmod +x surreal
./surreal version
tar -zcvf surreal-${{ needs.prepare-vars.outputs.name }}.linux-arm64.tgz surreal
echo $(shasum -a 256 surreal-${{ needs.prepare-vars.outputs.name }}.linux-arm64.tgz | cut -f1 -d' ') > surreal-${{ needs.prepare-vars.outputs.name }}.linux-arm64.txt
# Windows amd64
- arch: x86_64-pc-windows-msvc
runner: windows-latest
file: surreal-${{ needs.prepare-vars.outputs.name }}.windows-amd64
build-step: |
set -x
# Prepare deps
vcpkg integrate install
# Build
features=storage-tikv
if [[ "${{ inputs.http-compression }}" == "true" ]]; then
features=${features},http-compression
fi
if [[ "${{ inputs.ml }}" == "true" ]]; then
features=${features},ml
fi
cargo build --features $features --release --locked --target x86_64-pc-windows-msvc
# Package
./target/x86_64-pc-windows-msvc/release/surreal.exe version
cp target/x86_64-pc-windows-msvc/release/surreal.exe surreal-${{ needs.prepare-vars.outputs.name }}.windows-amd64.exe
echo $(shasum -a 256 surreal-${{ needs.prepare-vars.outputs.name }}.windows-amd64.exe | cut -f1 -d' ') > surreal-${{ needs.prepare-vars.outputs.name }}.windows-amd64.txt
runs-on: ${{ matrix.runner }}
steps:
- name: Checkout sources
uses: actions/checkout@v4
with:
ref: ${{ needs.prepare-vars.outputs.git-ref }}
- name: Checkout docker
uses: actions/checkout@v4
with:
path: _docker
# Replace docker files. It allows us to test new Dockerfiles with workflow_dispatch and a custom git ref.
# When triggered by a push or a schedule, this git ref will be the same as 'inputs.git-ref'
- name: Replace docker files
env:
GH_TOKEN: ${{ github.token }}
run: |
rm -rf docker .dockerignore
mv _docker/docker .
mv _docker/.dockerignore .
rm -rf _docker
- name: Install stable toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: 1.71.1
targets: ${{ matrix.arch }}
- name: Output package versions
run: |
set -x
set +e
go version ; cargo version ; rustc --version ; cmake --version ; gcc --version ; g++ --version ; perl -v
- name: Build step
env:
SURREAL_BUILD_METADATA: ${{ needs.prepare-vars.outputs.build-metadata }}
run: ${{ matrix.build-step }}
- name: Upload artifacts
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.file }}
path: |
surreal
${{ matrix.file }}.tgz
${{ matrix.file }}.txt
${{ matrix.file }}.exe
publish:
name: Publish crate and artifacts binaries
needs: [prepare-vars, test, lint, build, docker-build]
if: ${{ inputs.publish }}
environment: ${{ inputs.environment }}
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v4
with:
ref: ${{ needs.prepare-vars.outputs.git-ref }}
- name: Install stable toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: 1.74.1
- name: Install release-plz
run: cargo install --force --locked --version 0.3.30 release-plz
- name: Install a TOML parser
if: ${{ inputs.environment == 'beta' }}
run: cargo install --force --locked --version 0.8.1 taplo-cli
- name: Create a temporary branch
run: git checkout -b crate
- name: Configure release-plz
run: |
cat << EOF > /tmp/release-plz.toml
[workspace]
changelog_update = false
git_release_enable = false
semver_check = false
git_tag_enable = false
EOF
- name: Patch beta crate version
if: ${{ inputs.environment == 'beta' }}
run: |
set -x
# Derive crate version
currentVersion=$(/home/runner/.cargo/bin/taplo get -f lib/Cargo.toml "package.version")
major=$(echo $currentVersion | tr "." "\n" | sed -n 1p)
minor=$(echo $currentVersion | tr "." "\n" | sed -n 2p)
betaNum=$(echo $currentVersion | tr "." "\n" | sed -n 4p)
version=${major}.${minor}.$(($betaNum - 1))
# Update crate version
sed -i "s#^version = \".*\"#version = \"${version}\"#" lib/Cargo.toml
- name: Patch nightly crate version
if: ${{ inputs.environment == 'nightly' }}
run: |
# Get the date and time of the last commit
date=$(git show --no-patch --format=%ad --date=format:%Y%m%d)
time=$(git show --no-patch --format=%ad --date=format:%H%M%S)
# Update the version to a nightly one
# This sets the nightly version to something like `1.20231117.1130416`
# The 1 at the beginning of the patch number is just so it never starts with zero
sed -i "s#^version = \"\([[:digit:]]*\)\..*\"#version = \"\1.${date}.1${time}\"#" lib/Cargo.toml
- name: Patch crate name and description
if: ${{ inputs.environment == 'nightly' || inputs.environment == 'beta' }}
run: |
set -x
# Patch crate name
sed -i "0,/surrealdb/s//surrealdb-${{ inputs.environment }}/" lib/Cargo.toml
# Patch the description
sed -i "s#^description = \".*\"#description = \"A ${{ inputs.environment }} release of the surrealdb crate\"#" lib/Cargo.toml
# Temporarily commit patches
# These should not be pushed back to the repo
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config user.name "github-actions[bot]"
git commit -am "Name and version patches"
- name: Publish the crate
env:
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
run: /home/runner/.cargo/bin/release-plz release --config /tmp/release-plz.toml
- name: Download artifacts
uses: actions/download-artifact@v3
with:
path: artifacts
- name: Publish release
uses: softprops/action-gh-release@v1
if: ${{ inputs.create-release }}
with:
name: "Release ${{ needs.prepare-vars.outputs.git-ref }}"
tag_name: ${{ needs.prepare-vars.outputs.git-ref }}
prerelease: ${{ inputs.environment == 'beta' || inputs.environment == 'nightly' }}
fail_on_unmatched_files: true
files: |
LICENSE
artifacts/surreal-${{ needs.prepare-vars.outputs.git-ref }}.*/*.tgz
artifacts/surreal-${{ needs.prepare-vars.outputs.git-ref }}.*/*.exe
- name: Configure AWS
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-east-2
aws-access-key-id: ${{ secrets.AMAZON_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.AMAZON_SECRET_KEY }}
- name: Set latest release version
if: ${{ inputs.create-release && inputs.latest }}
run: |
echo ${{ needs.prepare-vars.outputs.git-ref }} > latest.txt
aws s3 cp --cache-control 'no-store' latest.txt s3://download.surrealdb.com/latest.txt
- name: Set latest beta version
if: ${{ inputs.publish && inputs.environment == 'beta' }}
run: |
echo ${{ needs.prepare-vars.outputs.git-ref }} > beta.txt
aws s3 cp --cache-control 'no-store' beta.txt s3://download.surrealdb.com/beta.txt
- name: Publish binaries
run: |
for file in artifacts/**/*.{tgz,txt,exe}; do
aws s3 cp --cache-control 'no-store' $file s3://download.surrealdb.com/${{ needs.prepare-vars.outputs.name }}/
done
docker-publish:
name: Publish Docker images
needs: [prepare-vars, publish]
uses: ./.github/workflows/reusable_docker.yml
with:
git-ref: ${{ needs.prepare-vars.outputs.git-ref }}
tag-prefix: ${{ needs.prepare-vars.outputs.name }}
latest: ${{ inputs.latest }}
build: false
push: true
secrets: inherit
package-macos:
name: Package and publish macOS universal binary
needs: [prepare-vars, publish]
runs-on: macos-latest
env:
FILE: surreal-${{ needs.prepare-vars.outputs.name }}.darwin-universal
steps:
- name: Download amd64 binary
uses: actions/download-artifact@v3
with:
name: surreal-${{ needs.prepare-vars.outputs.name }}.darwin-amd64
path: amd64
- name: Download arm64 binary
uses: actions/download-artifact@v3
with:
name: surreal-${{ needs.prepare-vars.outputs.name }}.darwin-arm64
path: arm64
- name: Configure AWS
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-east-2
aws-access-key-id: ${{ secrets.AMAZON_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.AMAZON_SECRET_KEY }}
- name: Package universal MacOS binary
run: |
lipo -create -output surreal amd64/surreal arm64/surreal
chmod +x surreal
tar -zcvf $FILE.tgz surreal
echo $(shasum -a 256 $FILE.tgz | cut -f1 -d' ') > $FILE.txt
- name: Publish universal MacOS binary
if: ${{ inputs.publish }}
run: |
aws s3 cp --cache-control 'no-store' $FILE.tgz s3://download.surrealdb.com/${{ needs.prepare-vars.outputs.name }}/
aws s3 cp --cache-control 'no-store' $FILE.txt s3://download.surrealdb.com/${{ needs.prepare-vars.outputs.name }}/
propagate:
name: Propagate binaries to all regions
if: ${{ inputs.publish }}
needs: [publish, package-macos]
runs-on: ubuntu-latest
steps:
- name: Configure AWS
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-east-2
aws-access-key-id: ${{ secrets.AMAZON_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.AMAZON_SECRET_KEY }}
- name: Distribute binaries
run: |
regions=("af-south-1" "ap-east-1" "ap-south-1" "ap-southeast-1" "ap-southeast-2" "ca-central-1" "eu-central-1" "eu-west-2" "me-south-1" "sa-east-1" "us-west-2")
for region in ${regions[@]}; do
aws s3 sync --delete --storage-class INTELLIGENT_TIERING --source-region eu-west-2 --region ${region} s3://download.surrealdb.com s3://download.${region}.surrealdb.com
done