[bench] New benchmarks against different datastores (#2956)

This commit is contained in:
Salvador Girones Gil 2023-11-18 14:55:01 +01:00 committed by GitHub
parent 7a34452262
commit b0be22360e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 958 additions and 54 deletions

View file

@ -5,49 +5,212 @@ on:
push:
branches:
- main
pull_request:
defaults:
run:
shell: bash
#
# The bench jobs will:
# 1. Run the benchmark and save the results as a baseline named "current"
# 2. Download the following baselines from S3
# - The latest baseline from the main branch
# - The latest baseline from the current branch
# 3. Compare the current benchmark results vs the baselines
# 4. Save the comparison as an artifact
# 5. Upload the current benchmark results as a new baseline and update the latest baseline for the current branch
#
jobs:
bench:
name: Bench library
runs-on: ubuntu-latest
common:
name: Bench common
runs-on:
- self-hosted
- benches
timeout-minutes: 60
steps:
- name: Install stable toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: 1.71.1
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-east-1
- name: Install dependencies
run: |
sudo apt-get -qq -y update
cargo install --quiet critcmp
- name: Checkout baseline
uses: actions/checkout@v3
with:
ref: ${{ github.base_ref }}
- name: Install cargo-make
run: cargo install --debug cargo-make
- name: Benchmark baseline
run: cargo make bench-baseline
sudo apt-get -qq -y install clang curl
cargo install --quiet critcmp cargo-make
- name: Checkout changes
uses: actions/checkout@v3
- name: Run benchmark
run: |
cargo make ci-bench -- --save-baseline current
- name: Copy results from AWS S3 bucket
run: |
BRANCH_NAME=$(echo ${{ github.head_ref || github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')
- name: Benchmark changes
run: cargo make bench-changes
aws s3 sync s3://${{ secrets.AWS_S3_GITHUB_ACTIONS_BUCKET_NAME }}/bench-results/${{ github.job }}/main/latest bench-results-main || true
aws s3 sync s3://${{ secrets.AWS_S3_GITHUB_ACTIONS_BUCKET_NAME }}/bench-results/${{ github.job }}/$BRANCH_NAME/latest bench-results-previous || true
- name: Benchmark results
run: cargo make bench-compare
- name: Compare current benchmark results vs baseline
run: |
mkdir -p bench-results
critcmp current bench-results-main/${{ matrix.target }}.json bench-results-previous/${{ matrix.target }}.json | tee bench-results/${{ github.job }}-comparison.txt
# Create a summary of the comparison
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
cat bench-results/${{ github.job }}-comparison.txt >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
- name: Save results as artifact
uses: actions/upload-artifact@v1
with:
name: Benchmark Results
path: benchmark_results
name: ${{ github.job }}-comparison.txt
path: bench-results/${{ github.job }}-comparison.txt
- name: Copy results to AWS S3 bucket
run: |
BRANCH_NAME=$(echo ${{ github.head_ref || github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')
cargo make ci-bench -- --load-baseline current --save-baseline previous
critcmp --export previous > bench-results/${{ matrix.target }}.json
aws s3 sync bench-results s3://${{ secrets.AWS_S3_GITHUB_ACTIONS_BUCKET_NAME }}/bench-results/${{ github.job }}/$BRANCH_NAME/${{ github.run_id }}
aws s3 sync bench-results s3://${{ secrets.AWS_S3_GITHUB_ACTIONS_BUCKET_NAME }}/bench-results/${{ github.job }}/$BRANCH_NAME/latest
engines:
name: Benchmark engines
runs-on:
- self-hosted
- benches
timeout-minutes: 60
permissions:
id-token: write
contents: read
strategy:
fail-fast: false
matrix:
include:
- target: "lib-mem"
features: "kv-mem"
- target: "lib-rocksdb"
features: "kv-rocksdb"
- target: "lib-fdb"
features: "kv-fdb-7_1"
- target: "sdk-mem"
features: "kv-mem"
- target: "sdk-rocksdb"
features: "kv-rocksdb"
- target: "sdk-fdb"
features: "kv-fdb-7_1"
# This one fails because the server consumes too much memory and the kernel kills it. I tried with instances up to 16GB of RAM.
# - target: "sdk-ws"
# features: "protocol-ws"
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install stable toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: 1.71.1
- name: Setup cache
uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-east-1
- name: Install dependencies
run: |
sudo apt-get -qq -y update
sudo apt-get -qq -y install clang curl
cargo install --quiet critcmp cargo-make
# Install FoundationDB if needed
- name: Setup FoundationDB
uses: foundationdb-rs/foundationdb-actions-install@v2.1.0
if: ${{ matrix.target == 'lib-fdb' || matrix.target == 'sdk-fdb' }}
with:
version: "7.1.30"
# Run SurrealDB in the background if needed
- name: Build and start SurrealDB
if: ${{ matrix.target == 'sdk-ws' }}
run: |
cargo make build
# Kill any potential previous instance of the server. The runner may be reused.
pkill -9 surreal || true
./target/release/surreal start 2>&1 >surrealdb.log &
set +e
echo "Waiting for surreal to be ready..."
tries=0
while [[ $tries < 5 ]]; do
./target/release/surreal is-ready 2>/dev/null && echo "Ready!" && exit 0 || sleep 1
tries=$((tries + 1))
done
echo "#####"
echo "SurrealDB server failed to start!"
echo "#####"
cat surrealdb.log
exit 1
- name: Run benchmark
env:
BENCH_FEATURES: "${{ matrix.features }}"
BENCH_DURATION: 60
BENCH_WORKER_THREADS: 2
run: |
cargo make bench-${{ matrix.target }} -- --save-baseline current
# Kill surreal server if it's running
pkill -9 surreal || true
- name: Copy results from AWS S3 bucket
run: |
BRANCH_NAME=$(echo ${{ github.head_ref || github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')
aws s3 sync s3://${{ secrets.AWS_S3_GITHUB_ACTIONS_BUCKET_NAME }}/bench-results/${{ github.job }}/main/latest bench-results-main || true
aws s3 sync s3://${{ secrets.AWS_S3_GITHUB_ACTIONS_BUCKET_NAME }}/bench-results/${{ github.job }}/$BRANCH_NAME/latest bench-results-previous || true
- name: Compare current benchmark results vs baseline
run: |
mkdir -p bench-results
critcmp current bench-results-main/${{ matrix.target }}.json bench-results-previous/${{ matrix.target }}.json | tee bench-results/${{ matrix.target }}-comparison.txt
# Create a summary of the comparison
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
cat bench-results/${{ matrix.target }}-comparison.txt >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
- name: Save results as artifact
uses: actions/upload-artifact@v1
with:
name: ${{ matrix.target }}-comparison.txt
path: bench-results/${{ matrix.target }}-comparison.txt
- name: Copy results to AWS S3 bucket
env:
BENCH_FEATURES: "${{ matrix.features }}"
run: |
BRANCH_NAME=$(echo ${{ github.head_ref || github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')
cargo make bench-${{ matrix.target }} -- --load-baseline current --save-baseline previous
critcmp --export previous > bench-results/${{ matrix.target }}.json
aws s3 sync bench-results s3://${{ secrets.AWS_S3_GITHUB_ACTIONS_BUCKET_NAME }}/bench-results/${{ github.job }}/$BRANCH_NAME/${{ github.run_id }}
aws s3 sync bench-results s3://${{ secrets.AWS_S3_GITHUB_ACTIONS_BUCKET_NAME }}/bench-results/${{ github.job }}/$BRANCH_NAME/latest

View file

@ -207,34 +207,60 @@ command = "cargo"
args = ["build", "--locked", "--no-default-features", "--features", "storage-mem"]
#
# Benchmarks
# Benchmarks - Common
#
[tasks.bench-run-baseline]
[tasks.ci-bench]
category = "CI - BENCHMARK"
command = "cargo"
args = ["bench", "--quiet", "--package", "surrealdb", "--no-default-features", "--features", "kv-mem,scripting,http", "--", "--save-baseline", "baseline"]
args = ["bench", "--quiet", "--package", "surrealdb", "--no-default-features", "--features", "kv-mem,scripting,http", "${@}"]
[tasks.bench-save-baseline]
category = "CI - BENCHMARK"
script = "cp -r target/criterion /tmp/criterion"
#
# Benchmarks - SDB - Per Target
#
[env]
BENCH_WORKER_THREADS = { value = "1", condition = { env_not_set = ["BENCH_WORKER_THREADS"] } }
BENCH_NUM_OPS = { value = "1000", condition = { env_not_set = ["BENCH_NUM_OPS"] } }
BENCH_DURATION = { value = "30", condition = { env_not_set = ["BENCH_DURATION"] } }
BENCH_SAMPLE_SIZE = { value = "10", condition = { env_not_set = ["BENCH_SAMPLE_SIZE"] } }
BENCH_FEATURES = { value = "protocol-ws,kv-mem,kv-rocksdb,kv-fdb-7_1", condition = { env_not_set = ["BENCH_FEATURES"] } }
[tasks.bench-restore-baseline]
category = "CI - BENCHMARK"
script = "mkdir -p target && cp -r /tmp/criterion target/criterion"
[tasks.bench-baseline]
category = "CI - BENCHMARK"
run_task = { name = ["bench-run-baseline", "bench-save-baseline"] }
[tasks.bench-run-changes]
category = "CI - BENCHMARK"
[tasks.bench-target]
private = true
category = "CI - BENCHMARK - SurrealDB Target"
command = "cargo"
args = ["bench", "--quiet", "--package", "surrealdb", "--no-default-features", "--features", "kv-mem,scripting,http", "--", "--save-baseline", "changes"]
args = ["bench", "--package", "surrealdb", "--bench", "sdb", "--no-default-features", "--features", "${BENCH_FEATURES}", "${@}"]
[tasks.bench-changes]
category = "CI - BENCHMARK"
run_task = { name = ["bench-restore-baseline", "bench-run-changes"] }
[tasks.bench-lib-mem]
category = "CI - BENCHMARK - SurrealDB Target"
env = { BENCH_DATASTORE_TARGET = "lib-mem" }
run_task = { name = ["bench-target"] }
[tasks.bench-compare]
category = "CI - BENCHMARK"
script = "critcmp baseline changes | tee benchmark_results"
[tasks.bench-lib-rocksdb]
category = "CI - BENCHMARK - SurrealDB Target"
env = { BENCH_DATASTORE_TARGET = "lib-rocksdb" }
run_task = { name = ["bench-target"] }
[tasks.bench-lib-fdb]
category = "CI - BENCHMARK - SurrealDB Target"
env = { BENCH_DATASTORE_TARGET = "lib-fdb" }
run_task = { name = ["bench-target"] }
[tasks.bench-sdk-mem]
category = "CI - BENCHMARK - SurrealDB Target"
env = { BENCH_DATASTORE_TARGET = "sdk-mem" }
run_task = { name = ["bench-target"] }
[tasks.bench-sdk-rocksdb]
category = "CI - BENCHMARK - SurrealDB Target"
env = { BENCH_DATASTORE_TARGET = "sdk-rocksdb" }
run_task = { name = ["bench-target"] }
[tasks.bench-sdk-fdb]
category = "CI - BENCHMARK - SurrealDB Target"
env = { BENCH_DATASTORE_TARGET = "sdk-fdb" }
run_task = { name = ["bench-target"] }
[tasks.bench-sdk-ws]
category = "CI - BENCHMARK - SurrealDB Target"
env = { BENCH_DATASTORE_TARGET = "sdk-ws" }
run_task = { name = ["bench-target"] }

View file

@ -68,7 +68,7 @@ args = ["clean"]
[tasks.bench]
category = "LOCAL USAGE"
command = "cargo"
args = ["bench", "--package", "surrealdb", "--no-default-features", "--features", "kv-mem,http,scripting"]
args = ["bench", "--package", "surrealdb", "--no-default-features", "--features", "kv-mem,http,scripting", "--", "${@}"]
# Run
[tasks.run]

View file

@ -168,3 +168,7 @@ harness = false
[[bench]]
name = "move_vs_clone"
harness = false
[[bench]]
name = "sdb"
harness = false

View file

@ -1,22 +1,63 @@
# Benchmarks
This directory contains some micro-benchmarks that can help objectively
establish the performance implications of a change.
establish the performance implications of a change, and also benchmarks that
test the performance of different datastores using both the library and the SDK
## Manual usage
### Common
Execute the following command at the top level of the repository:
```console
cargo bench --package surrealdb --no-default-features --features kv-mem,scripting,http
$ cargo make bench
```
### Specific datastore
Execute the following commands at the top level of the repository:
* Memory datastore using the lib or the SDK
```console
$ cargo make bench-lib-mem
$ cargo make bench-sdk-mem
```
* RocksDB datastore using the lib or the SDK
```console
$ cargo make bench-lib-rocksdb
$ cargo make bench-sdk-rocksdb
```
* FoundationDB datastore using the lib or the SDK
* Start FoundationDB
```
$ docker run -ti -e FDB_NETWORKING_MODE=host --net=host foundationdb/foundationdb:7.1.30
```
* Run the benchmarks
```console
$ cargo make bench-lib-rocksdb
$ cargo make bench-sdk-rocksdb
```
* WebSocket remote server using the SDK
* Start SurrealDB server
```
$ cargo make build
$ ./target/release/surreal start
```
* Run the benchmarks
```console
$ cargo make bench-sdk-ws
```
## Profiling
Some of the benchmarks support CPU profiling:
```console
cargo bench --package surrealdb --no-default-features --features kv-mem,scripting,http -- --profile-time=5
cargo make bench --profile-time=5
```
Once complete, check the `target/criterion/**/profile/flamegraph.svg` files.
Once complete, check the `target/criterion/**/profile/flamegraph.svg` files.

17
lib/benches/sdb.rs Normal file
View file

@ -0,0 +1,17 @@
mod sdb_benches;
use criterion::{criterion_group, criterion_main, Criterion};
use pprof::criterion::{Output, PProfProfiler};
fn bench(c: &mut Criterion) {
let target = std::env::var("BENCH_DATASTORE_TARGET").unwrap_or("lib-mem".to_string());
sdb_benches::benchmark_group(c, target);
}
criterion_group!(
name = benches;
config = Criterion::default().with_profiler(PProfProfiler::new(1000, Output::Flamegraph(None)));
targets = bench
);
criterion_main!(benches);

View file

@ -0,0 +1,75 @@
use criterion::{Criterion, Throughput};
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use surrealdb::dbs::Session;
use surrealdb::kvs::Datastore;
mod routines;
static DB: OnceLock<Arc<Datastore>> = OnceLock::new();
pub(super) async fn init(target: &str) {
match target {
#[cfg(feature = "kv-mem")]
"lib-mem" => {
let _ = DB.set(Arc::new(Datastore::new("memory").await.unwrap()));
}
#[cfg(feature = "kv-rocksdb")]
"lib-rocksdb" => {
let path = format!(
"rocksdb://lib-rocksdb-{}.db",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis()
);
println!("\n### Using path: {} ###\n", path);
let ds = Datastore::new(&path).await.unwrap();
ds.execute("INFO FOR DB", &Session::owner().with_ns("ns").with_db("db"), None)
.await
.expect("Unable to execute the query");
let _ = DB.set(Arc::new(ds));
}
#[cfg(feature = "kv-fdb")]
"lib-fdb" => {
let ds = Datastore::new("fdb:///etc/foundationdb/fdb.cluster").await.unwrap();
// Verify it can connect to the FDB cluster
ds.execute("INFO FOR DB", &Session::owner().with_ns("ns").with_db("db"), None)
.await
.expect("Unable to connect to FDB cluster");
let _ = DB.set(Arc::new(ds));
}
_ => panic!("Unknown target: {}", target),
}
}
pub(super) fn benchmark_group(c: &mut Criterion, target: String) {
let num_ops = super::NUM_OPS.clone();
let runtime = super::rt();
runtime.block_on(async { init(&target).await });
let mut group = c.benchmark_group(target);
group.measurement_time(Duration::from_secs(super::DURATION_SECS.clone()));
group.sample_size(super::SAMPLE_SIZE.clone());
group.throughput(Throughput::Elements(1));
group.bench_function("reads", |b| {
routines::bench_routine(
b,
DB.get().unwrap().clone(),
routines::Read::new(super::rt()),
num_ops,
)
});
group.bench_function("creates", |b| {
routines::bench_routine(
b,
DB.get().unwrap().clone(),
routines::Create::new(super::rt()),
num_ops,
)
});
group.finish();
}

View file

@ -0,0 +1,82 @@
use std::sync::Arc;
use surrealdb::dbs::Session;
use surrealdb::{kvs::Datastore, sql::Id};
use tokio::{runtime::Runtime, task::JoinSet};
pub struct Create {
runtime: &'static Runtime,
table_name: String,
}
impl Create {
pub fn new(runtime: &'static Runtime) -> Self {
Self {
runtime,
table_name: format!("table_{}", Id::rand().to_string()),
}
}
}
impl super::Routine for Create {
fn setup(&self, ds: Arc<Datastore>, session: Session, _num_ops: usize) {
self.runtime.block_on(async {
// Create table
let mut res = ds
.execute(format!("DEFINE TABLE {}", &self.table_name).as_str(), &session, None)
.await
.expect("[setup] define table failed");
let _ = res.remove(0).output().expect("[setup] the create operation returned no value");
});
}
fn run(&self, ds: Arc<Datastore>, session: Session, num_ops: usize) {
self.runtime.block_on(async {
// Spawn one task for each operation
let mut tasks = JoinSet::default();
for _ in 0..num_ops {
let ds = ds.clone();
let session = session.clone();
let table_name = self.table_name.clone();
tasks.spawn_on(
async move {
let mut res = criterion::black_box(
ds.execute(
format!("CREATE {} SET field = '{}'", &table_name, Id::rand())
.as_str(),
&session,
None,
)
.await
.expect("[setup] create record failed"),
);
let res = res
.remove(0)
.output()
.expect("[setup] the create operation returned no value");
if res.is_none_or_null() {
panic!("[setup] Record not found");
}
},
self.runtime.handle(),
);
}
while let Some(task) = tasks.join_next().await {
task.unwrap();
}
});
}
fn cleanup(&self, ds: Arc<Datastore>, session: Session, _num_ops: usize) {
self.runtime.block_on(async {
let mut res = ds
.execute(format!("REMOVE TABLE {}", self.table_name).as_str(), &session, None)
.await
.expect("[cleanup] remove table failed");
let _ =
res.remove(0).output().expect("[cleanup] the remove operation returned no value");
});
}
}

View file

@ -0,0 +1,55 @@
use std::sync::Arc;
use criterion::measurement::WallTime;
use criterion::Bencher;
use surrealdb::dbs::Session;
use surrealdb::kvs::Datastore;
mod create;
pub(super) use create::*;
mod read;
pub(super) use read::*;
/// Routine trait for the benchmark routines.
///
/// The `setup` function is called once before the benchmark starts. It's used to prepare the database for the benchmark.
/// The `run` function is called for each iteration of the benchmark.
/// The `cleanup` function is called once after the benchmark ends. It's used to clean up the database after the benchmark.
pub(super) trait Routine {
fn setup(&self, ds: Arc<Datastore>, session: Session, num_ops: usize);
fn run(&self, ds: Arc<Datastore>, session: Session, num_ops: usize);
fn cleanup(&self, ds: Arc<Datastore>, session: Session, num_ops: usize);
}
/// Execute the setup, benchmark the `run` function, and execute the cleanup.
pub(super) fn bench_routine<R>(
b: &mut Bencher<'_, WallTime>,
ds: Arc<Datastore>,
routine: R,
num_ops: usize,
) where
R: Routine,
{
// Run the runtime and return the duration, accounting for the number of operations on each run
b.iter_custom(|iters| {
let num_ops = num_ops.clone();
// Total time spent running the actual benchmark run for all iterations
let mut total = std::time::Duration::from_secs(0);
let session = Session::owner().with_ns("test").with_db("test");
for _ in 0..iters {
// Setup
routine.setup(ds.clone(), session.clone(), num_ops);
// Run and time the routine
let now = std::time::Instant::now();
routine.run(ds.clone(), session.clone(), num_ops);
total += now.elapsed();
// Cleanup the database
routine.cleanup(ds.clone(), session.clone(), num_ops);
}
total.div_f32(num_ops as f32)
});
}

View file

@ -0,0 +1,126 @@
use std::sync::Arc;
use surrealdb::dbs::Session;
use surrealdb::{kvs::Datastore, sql::Id};
use tokio::{runtime::Runtime, task::JoinSet};
pub struct Read {
runtime: &'static Runtime,
table_name: String,
}
impl Read {
pub fn new(runtime: &'static Runtime) -> Self {
Self {
runtime,
table_name: format!("table_{}", Id::rand().to_string()),
}
}
}
impl super::Routine for Read {
fn setup(&self, ds: Arc<Datastore>, session: Session, num_ops: usize) {
self.runtime.block_on(async {
// Create table
let mut res = ds
.execute(format!("DEFINE TABLE {}", &self.table_name).as_str(), &session, None)
.await
.expect("[setup] define table failed");
let _ = res.remove(0).output().expect("[setup] the create operation returned no value");
// Spawn one task for each operation
let mut tasks = JoinSet::default();
for task_id in 0..num_ops {
let ds = ds.clone();
let session = session.clone();
let table_name = self.table_name.clone();
tasks.spawn_on(
async move {
let mut res = ds
.execute(
format!(
"CREATE {}:{} SET field = '{}'",
&table_name,
task_id,
Id::rand()
)
.as_str(),
&session,
None,
)
.await
.expect("[setup] create record failed");
let res = res
.remove(0)
.output()
.expect("[setup] the create operation returned no value");
if res.is_none_or_null() {
panic!("[setup] Record not found");
}
},
self.runtime.handle(),
);
}
while let Some(task) = tasks.join_next().await {
task.unwrap();
}
});
}
fn run(&self, ds: Arc<Datastore>, session: Session, num_ops: usize) {
self.runtime.block_on(async {
// Spawn one task for each operation
let mut tasks = JoinSet::default();
for task_id in 0..num_ops {
let ds = ds.clone();
let session = session.clone();
let table_name = self.table_name.clone();
tasks.spawn_on(
async move {
let mut res = criterion::black_box(
ds.execute(
format!(
"SELECT * FROM {}:{} WHERE field = '{}'",
&table_name,
task_id,
Id::rand()
)
.as_str(),
&session,
None,
)
.await
.expect("[run] select operation failed"),
);
let res = res
.remove(0)
.output()
.expect("[run] the select operation returned no value");
if res.is_none_or_null() {
panic!("[run] Record not found");
}
},
self.runtime.handle(),
);
}
while let Some(task) = tasks.join_next().await {
task.unwrap();
}
});
}
fn cleanup(&self, ds: Arc<Datastore>, session: Session, _num_ops: usize) {
self.runtime.block_on(async {
let mut res = ds
.execute(format!("REMOVE TABLE {}", self.table_name).as_str(), &session, None)
.await
.expect("[cleanup] remove table failed");
let _ =
res.remove(0).output().expect("[cleanup] the remove operation returned no value");
});
}
}

View file

@ -0,0 +1,38 @@
use criterion::Criterion;
use once_cell::sync::Lazy;
use std::sync::OnceLock;
use tokio::runtime::Runtime;
mod lib;
mod sdk;
static NUM_OPS: Lazy<usize> =
Lazy::new(|| std::env::var("BENCH_NUM_OPS").unwrap_or("1000".to_string()).parse().unwrap());
static DURATION_SECS: Lazy<u64> =
Lazy::new(|| std::env::var("BENCH_DURATION").unwrap_or("30".to_string()).parse().unwrap());
static SAMPLE_SIZE: Lazy<usize> =
Lazy::new(|| std::env::var("BENCH_SAMPLE_SIZE").unwrap_or("30".to_string()).parse().unwrap());
static WORKER_THREADS: Lazy<usize> =
Lazy::new(|| std::env::var("BENCH_WORKER_THREADS").unwrap_or("1".to_string()).parse().unwrap());
static RUNTIME: OnceLock<Runtime> = OnceLock::new();
fn rt() -> &'static Runtime {
RUNTIME.get_or_init(|| {
tokio::runtime::Builder::new_multi_thread()
.worker_threads(WORKER_THREADS.clone())
.enable_all()
.build()
.unwrap()
})
}
/// Create a benchmark group for the given target.
pub(super) fn benchmark_group(c: &mut Criterion, target: String) {
println!("### Benchmark config: target={}, num_ops={}, duration={}, sample_size={}, worker_threads={} ###", target, *NUM_OPS, *DURATION_SECS, *SAMPLE_SIZE, *WORKER_THREADS);
match &target {
t if t.starts_with("lib") => lib::benchmark_group(c, target),
t if t.starts_with("sdk") => sdk::benchmark_group(c, target),
t => panic!("Target '{}' not supported.", t),
}
}

View file

@ -0,0 +1,69 @@
use criterion::{Criterion, Throughput};
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};
use std::time::Duration;
use surrealdb::{engine::any::Any, sql::Id, Surreal};
mod routines;
static DB: Lazy<Surreal<Any>> = Lazy::new(Surreal::init);
#[derive(Debug, Clone, Serialize, Deserialize)]
struct Record {
field: Id,
}
pub(super) async fn init(target: &str) {
match target {
#[cfg(feature = "kv-mem")]
"sdk-mem" => {
DB.connect("memory").await.unwrap();
}
#[cfg(feature = "kv-rocksdb")]
"sdk-rocksdb" => {
let path = format!(
"rocksdb://sdk-rocksdb-{}.db",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis()
);
println!("\n### Using path: {} ###\n", path);
DB.connect(&path).await.unwrap();
}
#[cfg(feature = "kv-fdb")]
"sdk-fdb" => {
DB.connect("fdb:///etc/foundationdb/fdb.cluster").await.unwrap();
// Verify it can connect to the FDB cluster
DB.health().await.expect("fdb cluster is unavailable");
}
#[cfg(feature = "protocol-ws")]
"sdk-ws" => {
DB.connect("ws://localhost:8000").await.unwrap();
}
_ => panic!("Unknown target: {}", target),
};
DB.use_ns("test").use_db("test").await.unwrap();
}
pub(super) fn benchmark_group(c: &mut Criterion, target: String) {
let num_ops = super::NUM_OPS.clone();
let runtime = super::rt();
runtime.block_on(async { init(&target).await });
let mut group = c.benchmark_group(target);
group.measurement_time(Duration::from_secs(super::DURATION_SECS.clone()));
group.sample_size(super::SAMPLE_SIZE.clone());
group.throughput(Throughput::Elements(1));
group.bench_function("reads", |b| {
routines::bench_routine(b, &DB, routines::Read::new(super::rt()), num_ops)
});
group.bench_function("creates", |b| {
routines::bench_routine(b, &DB, routines::Create::new(super::rt()), num_ops)
});
group.finish();
}

View file

@ -0,0 +1,67 @@
use surrealdb::{engine::any::Any, sql::Id, Surreal};
use tokio::{runtime::Runtime, task::JoinSet};
use crate::sdb_benches::sdk::Record;
pub struct Create {
runtime: &'static Runtime,
table_name: String,
}
impl Create {
pub fn new(runtime: &'static Runtime) -> Self {
Self {
runtime,
table_name: format!("table_{}", Id::rand().to_string()),
}
}
}
impl super::Routine for Create {
fn setup(&self, _client: &'static Surreal<Any>, _num_ops: usize) {}
fn run(&self, client: &'static Surreal<Any>, num_ops: usize) {
self.runtime.block_on(async {
let data = Record {
field: Id::rand(),
};
// Spawn one task for each operation
let mut tasks = JoinSet::default();
for _ in 0..num_ops {
let table_name = self.table_name.clone();
let data = data.clone();
tasks.spawn(async move {
let res: Vec<Record> = criterion::black_box(
client
.create(table_name)
.content(data)
.await
.expect("[run] record creation failed"),
);
assert_eq!(
res.len(),
1,
"[run] expected record creation to return 1 record, got {}",
res.len()
);
});
}
while let Some(task) = tasks.join_next().await {
task.unwrap();
}
});
}
fn cleanup(&self, client: &'static Surreal<Any>, _num_ops: usize) {
self.runtime.block_on(async {
client
.query(format!("REMOVE TABLE {}", self.table_name))
.await
.expect("[cleanup] remove table failed");
});
}
}

View file

@ -0,0 +1,50 @@
use criterion::{measurement::WallTime, Bencher};
use surrealdb::{engine::any::Any, Surreal};
mod create;
pub(super) use create::*;
mod read;
pub(super) use read::*;
/// Routine trait for the benchmark routines.
///
/// The `setup` function is called once before the benchmark starts. It's used to prepare the database for the benchmark.
/// The `run` function is called for each iteration of the benchmark.
/// The `cleanup` function is called once after the benchmark ends. It's used to clean up the database after the benchmark.
pub(super) trait Routine {
fn setup(&self, ds: &'static Surreal<Any>, num_ops: usize);
fn run(&self, ds: &'static Surreal<Any>, num_ops: usize);
fn cleanup(&self, ds: &'static Surreal<Any>, num_ops: usize);
}
/// Execute the setup, benchmark the `run` function, and execute the cleanup.
pub(super) fn bench_routine<R>(
b: &mut Bencher<'_, WallTime>,
db: &'static Surreal<Any>,
routine: R,
num_ops: usize,
) where
R: Routine,
{
// Run the runtime and return the duration, accounting for the number of operations on each run
b.iter_custom(|iters| {
let num_ops = num_ops.clone();
// Total time spent running the actual benchmark run for all iterations
let mut total = std::time::Duration::from_secs(0);
for _ in 0..iters {
// Setup
routine.setup(db, num_ops.clone());
// Run and time the routine
let now = std::time::Instant::now();
routine.run(db, num_ops.clone());
total += now.elapsed();
// Cleanup the database
routine.cleanup(db, num_ops);
}
total.div_f32(num_ops as f32)
});
}

View file

@ -0,0 +1,80 @@
use surrealdb::{engine::any::Any, sql::Id, Surreal};
use tokio::{runtime::Runtime, task::JoinSet};
use crate::sdb_benches::sdk::Record;
pub struct Read {
runtime: &'static Runtime,
table_name: String,
}
impl Read {
pub fn new(runtime: &'static Runtime) -> Self {
Self {
runtime,
table_name: format!("table_{}", Id::rand().to_string()),
}
}
}
impl super::Routine for Read {
fn setup(&self, client: &'static Surreal<Any>, num_ops: usize) {
self.runtime.block_on(async {
// Spawn one task for each operation
let mut tasks = JoinSet::default();
for task_id in 0..num_ops {
let table_name = self.table_name.clone();
tasks.spawn(async move {
let _: Option<Record> = client
.create((table_name, task_id as u64))
.content(Record {
field: Id::rand(),
})
.await
.expect("[setup] create record failed")
.expect("[setup] the create operation returned None");
});
}
while let Some(task) = tasks.join_next().await {
task.unwrap();
}
});
}
fn run(&self, client: &'static Surreal<Any>, num_ops: usize) {
self.runtime.block_on(async {
// Spawn one task for each operation
let mut tasks = JoinSet::default();
for task_id in 0..num_ops {
let table_name = self.table_name.clone();
tasks.spawn(async move {
let _: Option<Record> = criterion::black_box(
client
.select((table_name, task_id as u64))
.await
.expect("[run] select operation failed")
.expect("[run] the select operation returned None"),
);
});
}
while let Some(task) = tasks.join_next().await {
task.unwrap();
}
});
}
fn cleanup(&self, client: &'static Surreal<Any>, _num_ops: usize) {
self.runtime.block_on(async {
client
.query(format!("REMOVE table {}", self.table_name))
.await
.expect("[cleanup] remove table failed")
.check()
.unwrap();
});
}
}

View file

@ -5,6 +5,7 @@ use crate::kvs::Check;
use crate::kvs::Key;
use crate::kvs::Val;
use crate::vs::{u64_to_versionstamp, Versionstamp};
use foundationdb::options;
use futures::TryStreamExt;
use std::ops::Range;
use std::sync::Arc;
@ -86,10 +87,20 @@ impl Datastore {
let _fdbnet = (*FDBNET).clone();
match foundationdb::Database::from_path(path) {
Ok(db) => Ok(Datastore {
db,
_fdbnet,
}),
Ok(db) => {
db.set_option(options::DatabaseOption::TransactionRetryLimit(5)).map_err(|e| {
Error::Ds(format!("Unable to set transaction retry limit: {}", e))
})?;
db.set_option(options::DatabaseOption::TransactionTimeout(5000))
.map_err(|e| Error::Ds(format!("Unable to set transaction timeout: {}", e)))?;
db.set_option(options::DatabaseOption::TransactionMaxRetryDelay(500)).map_err(
|e| Error::Ds(format!("Unable to set transaction max retry delay: {}", e)),
)?;
Ok(Datastore {
db,
_fdbnet,
})
}
Err(e) => Err(Error::Ds(e.to_string())),
}
}