Testing - Add microbenchmarks for query parsing and execution. (#1878)
This commit is contained in:
Finn Bear
GitHub
parent
a76910e826
commit
46c5b4d7f1
6 changed files with 359 additions and 2 deletions
44
.github/workflows/bench.yml
vendored
Normal file
44
.github/workflows/bench.yml
vendored
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
name: Benchmark
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
bench:
|
||||
name: Bench library
|
||||
runs-on: ubuntu-20.04-16-cores
|
||||
steps:
|
||||
- name: Install stable toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get -qq -y update
|
||||
sudo apt-get -qq -y install protobuf-compiler libprotobuf-dev
|
||||
cargo install --quiet critcmp
|
||||
|
||||
- name: Checkout baseline
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{ github.base_ref }}
|
||||
|
||||
- name: Benchmark baseline
|
||||
run: cargo bench --quiet --package surrealdb --no-default-features --features kv-mem -- --save-baseline baseline
|
||||
|
||||
- name: Checkout changes
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Benchmark changes
|
||||
run: cargo bench --quiet --package surrealdb --no-default-features --features kv-mem -- --save-baseline changes
|
||||
|
||||
- name: Compare results
|
||||
run: critcmp baseline changes | tee benchmark_results
|
||||
|
||||
- name: Save results as artifact
|
||||
uses: actions/upload-artifact@v1
|
||||
with:
|
||||
name: Benchmark Results
|
||||
path: benchmark_results
|
||||
175
Cargo.lock
generated
175
Cargo.lock
generated
|
|
@ -264,6 +264,12 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anes"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
|
||||
|
||||
[[package]]
|
||||
name = "any_ascii"
|
||||
version = "0.3.2"
|
||||
|
|
@ -742,6 +748,12 @@ dependencies = [
|
|||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cast"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.79"
|
||||
|
|
@ -791,6 +803,33 @@ dependencies = [
|
|||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ciborium"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f"
|
||||
dependencies = [
|
||||
"ciborium-io",
|
||||
"ciborium-ll",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ciborium-io"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369"
|
||||
|
||||
[[package]]
|
||||
name = "ciborium-ll"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b"
|
||||
dependencies = [
|
||||
"ciborium-io",
|
||||
"half",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cipher"
|
||||
version = "0.4.4"
|
||||
|
|
@ -947,6 +986,42 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb"
|
||||
dependencies = [
|
||||
"anes",
|
||||
"atty",
|
||||
"cast",
|
||||
"ciborium",
|
||||
"clap",
|
||||
"criterion-plot",
|
||||
"itertools 0.10.5",
|
||||
"lazy_static",
|
||||
"num-traits",
|
||||
"oorandom",
|
||||
"plotters",
|
||||
"rayon",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"tinytemplate",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion-plot"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
|
||||
dependencies = [
|
||||
"cast",
|
||||
"itertools 0.10.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "critical-section"
|
||||
version = "1.1.1"
|
||||
|
|
@ -963,6 +1038,30 @@ dependencies = [
|
|||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"memoffset",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.15"
|
||||
|
|
@ -2251,6 +2350,15 @@ version = "2.5.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
version = "0.3.17"
|
||||
|
|
@ -2448,6 +2556,12 @@ version = "1.17.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
|
||||
|
||||
[[package]]
|
||||
name = "oorandom"
|
||||
version = "11.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
|
||||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.50"
|
||||
|
|
@ -2766,6 +2880,34 @@ version = "0.3.26"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
|
||||
|
||||
[[package]]
|
||||
name = "plotters"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
"plotters-backend",
|
||||
"plotters-svg",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "plotters-backend"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142"
|
||||
|
||||
[[package]]
|
||||
name = "plotters-svg"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f"
|
||||
dependencies = [
|
||||
"plotters-backend",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.17"
|
||||
|
|
@ -3093,6 +3235,28 @@ dependencies = [
|
|||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
"num_cpus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.16"
|
||||
|
|
@ -3793,6 +3957,7 @@ dependencies = [
|
|||
"bigdecimal",
|
||||
"bung",
|
||||
"chrono",
|
||||
"criterion",
|
||||
"dmp",
|
||||
"echodb",
|
||||
"env_logger 0.10.0",
|
||||
|
|
@ -4086,6 +4251,16 @@ dependencies = [
|
|||
"time-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinytemplate"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.6.0"
|
||||
|
|
|
|||
|
|
@ -102,6 +102,7 @@ ulid = { version = "1.0.0", features = ["serde"] }
|
|||
url = "2.3.1"
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = "0.4"
|
||||
env_logger = "0.10.0"
|
||||
temp-dir = "0.1.11"
|
||||
time = { version = "0.3.20", features = ["serde"] }
|
||||
|
|
@ -116,9 +117,17 @@ wasm-bindgen-futures = "0.4.34"
|
|||
ws_stream_wasm = "0.7.4"
|
||||
|
||||
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
|
||||
tokio = { version = "1.27.0", default-features = false, features = ["macros", "io-util", "io-std", "fs", "rt-multi-thread"] }
|
||||
tokio = { version = "1.27.0", default-features = false, features = ["macros", "io-util", "io-std", "fs", "rt-multi-thread", "time"] }
|
||||
tokio-tungstenite = { version = "0.18.0", optional = true }
|
||||
uuid = { version = "1.3.1", features = ["serde", "v4", "v7"] }
|
||||
|
||||
[lib]
|
||||
bench = false
|
||||
bench = false
|
||||
|
||||
[[bench]]
|
||||
name = "executor"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "parser"
|
||||
harness = false
|
||||
|
|
|
|||
12
lib/benches/README.md
Normal file
12
lib/benches/README.md
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# Benchmarks
|
||||
|
||||
This directory contains some micro-benchmarks that can help objectively
|
||||
establish the performance implications of a change.
|
||||
|
||||
## Manual usage
|
||||
|
||||
Execute the following command at the top level of the repository:
|
||||
|
||||
```console
|
||||
cargo bench --package surrealdb --no-default-features --features kv-mem
|
||||
```
|
||||
52
lib/benches/executor.rs
Normal file
52
lib/benches/executor.rs
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput};
|
||||
use surrealdb::{dbs::Session, kvs::Datastore};
|
||||
|
||||
macro_rules! query {
|
||||
($c: expr, $name: ident, $query: expr) => {
|
||||
query!($c, $name, "", $query);
|
||||
};
|
||||
($c: expr, $name: ident, $setup: expr, $query: expr) => {
|
||||
$c.bench_function(stringify!($name), |b| {
|
||||
let (dbs, ses) = futures::executor::block_on(async {
|
||||
let dbs = Datastore::new("memory").await.unwrap();
|
||||
let ses = Session::for_kv().with_ns("test").with_db("test");
|
||||
let setup = $setup;
|
||||
if !setup.is_empty() {
|
||||
dbs.execute(setup, &ses, None, false).await.unwrap();
|
||||
}
|
||||
(dbs, ses)
|
||||
});
|
||||
|
||||
b.iter(|| {
|
||||
futures::executor::block_on(async {
|
||||
black_box(dbs.execute(black_box($query), &ses, None, false).await).unwrap();
|
||||
});
|
||||
})
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
fn bench_executor(c: &mut Criterion) {
|
||||
let mut c = c.benchmark_group("executor");
|
||||
c.throughput(Throughput::Elements(1));
|
||||
query!(c, create_delete_simple, "CREATE person:one; DELETE person:one;");
|
||||
query!(c, select_simple_one, "CREATE person:tobie;", "SELECT * FROM person;");
|
||||
query!(c, select_simple_five, "CREATE person:one; CREATE person:two; CREATE person:three; CREATE person:four; CREATE person:five;", "SELECT * FROM person;");
|
||||
query!(c, select_future, "SELECT * FROM <future>{5};");
|
||||
query!(
|
||||
c,
|
||||
update_simple,
|
||||
"CREATE thing:one SET value = 0;",
|
||||
"UPDATE thing SET value = value + 1;"
|
||||
);
|
||||
query!(
|
||||
c,
|
||||
select_record_link,
|
||||
"CREATE person:one SET friend = person:two; CREATE person:two SET age = 30;",
|
||||
"SELECT * FROM person:one.friend.age;"
|
||||
);
|
||||
c.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_executor);
|
||||
criterion_main!(benches);
|
||||
65
lib/benches/parser.rs
Normal file
65
lib/benches/parser.rs
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput};
|
||||
|
||||
macro_rules! parser {
|
||||
($c: expr, $name: ident, $parser: path, $text: expr) => {
|
||||
$c.bench_function(stringify!($name), |b| {
|
||||
let text = $text;
|
||||
|
||||
b.iter(|| black_box($parser(black_box(text))).unwrap())
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
fn bench_parser(c: &mut Criterion) {
|
||||
let mut c = c.benchmark_group("parser");
|
||||
c.throughput(Throughput::Elements(1));
|
||||
parser!(c, select_simple, surrealdb::sql::parse, "SELECT * FROM person;");
|
||||
parser!(
|
||||
c,
|
||||
select_complex,
|
||||
surrealdb::sql::parse,
|
||||
"SELECT name, age, country FROM person WHERE hair = 'brown' AND is_vegetarian;"
|
||||
);
|
||||
parser!(c, transaction, surrealdb::sql::parse, "BEGIN TRANSACTION; UPDATE person:finn SET squirrels = 'yes'; SELECT * FROM person; COMMIT TRANSACTION;");
|
||||
parser!(c, datetime, surrealdb::sql::parse, "RETURN '2022-07-03T07:18:52.841147+02:00';");
|
||||
parser!(c, duration, surrealdb::sql::parse, "RETURN [100w, 5d, 20m, 2s];");
|
||||
parser!(
|
||||
c,
|
||||
casting_deep,
|
||||
surrealdb::sql::parse,
|
||||
"RETURN <float><float><float><float><float>1.0;"
|
||||
);
|
||||
parser!(
|
||||
c,
|
||||
json_geo,
|
||||
surrealdb::sql::parse,
|
||||
"RETURN { type: 'Point', coordinates: [-0.118092, 51.509865] };"
|
||||
);
|
||||
parser!(c, json_number, surrealdb::sql::json, "1.2345");
|
||||
parser!(
|
||||
c,
|
||||
json_small_object,
|
||||
surrealdb::sql::json,
|
||||
"{'key': true, 'number': 42.0, 'value': null}"
|
||||
);
|
||||
parser!(c, json_small_array, surrealdb::sql::json, "[1, false, null, 'foo']");
|
||||
parser!(
|
||||
c,
|
||||
json_large_array,
|
||||
surrealdb::sql::json,
|
||||
&format!("[{}]", (1..=100).map(|n| n.to_string()).collect::<Vec<_>>().join(", "))
|
||||
);
|
||||
parser!(
|
||||
c,
|
||||
json_large_object,
|
||||
surrealdb::sql::json,
|
||||
&format!(
|
||||
"{{{}}}",
|
||||
&(1..=100).map(|n| format!("'{n}': {n}")).collect::<Vec<_>>().join(", ")
|
||||
)
|
||||
);
|
||||
c.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench_parser);
|
||||
criterion_main!(benches);
|
||||
Loading…
Reference in a new issue