Testing - Add CPU profiling to microbenchmarks (#2000)

2023-05-16 15:31:30 -07:00 · 2023-05-16 15:31:30 -07:00 · 94a376adc5
commit 94a376adc5
parent 437fc231f1
7 changed files with 232 additions and 6 deletions
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@ -26,7 +26,7 @@ jobs:
          ref: ${{ github.base_ref }}

      - name: Benchmark baseline
-        run: cargo bench --quiet --package surrealdb --no-default-features --features kv-mem -- --save-baseline baseline
+        run: cargo bench --quiet --package surrealdb --no-default-features --features kv-mem,scripting,http -- --save-baseline baseline

      - name: Extract baseline
        run: cp -r target/criterion /tmp/criterion
@ -38,7 +38,7 @@ jobs:
        run: mkdir target && cp -r /tmp/criterion target/criterion

      - name: Benchmark changes
-        run: cargo bench --quiet --package surrealdb --no-default-features --features kv-mem -- --save-baseline changes
+        run: cargo bench --quiet --package surrealdb --no-default-features --features kv-mem,scripting,http -- --save-baseline changes

      - name: Compare results
        run: critcmp baseline changes | tee benchmark_results
--- a/Cargo.lock
+++ b/Cargo.lock
@ -202,6 +202,15 @@ dependencies = [
 "psl-types",
 ]

+[[package]]
+name = "addr2line"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97"
+dependencies = [
+ "gimli",
+]
+
 [[package]]
 name = "adler"
 version = "1.0.2"
@ -308,6 +317,12 @@ dependencies = [
 "password-hash",
 ]

+[[package]]
+name = "arrayvec"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
+
 [[package]]
 name = "async-channel"
 version = "1.8.0"
@ -512,6 +527,21 @@ dependencies = [
 "tokio",
 ]

+[[package]]
+name = "backtrace"
+version = "0.3.67"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca"
+dependencies = [
+ "addr2line",
+ "cc",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+]
+
 [[package]]
 name = "base64"
 version = "0.13.1"
@ -716,6 +746,12 @@ dependencies = [
 "serde",
 ]

+[[package]]
+name = "bytemuck"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea"
+
 [[package]]
 name = "byteorder"
 version = "1.4.3"
@ -968,6 +1004,15 @@ version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"

+[[package]]
+name = "cpp_demangle"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c76f98bdfc7f66172e6c7065f981ebb576ffc903fe4c0561d9f0c2509226dc6"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "cpufeatures"
 version = "0.2.6"
@ -1173,6 +1218,15 @@ dependencies = [
 "parking_lot_core 0.9.7",
 ]

+[[package]]
+name = "debugid"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d"
+dependencies = [
+ "uuid",
+]
+
 [[package]]
 name = "derive-new"
 version = "0.5.9"
@ -1376,6 +1430,18 @@ dependencies = [
 "log",
 ]

+[[package]]
+name = "findshlibs"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64"
+dependencies = [
+ "cc",
+ "lazy_static",
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "fixedbitset"
 version = "0.2.0"
@ -1707,6 +1773,12 @@ dependencies = [
 "wasm-bindgen",
 ]

+[[package]]
+name = "gimli"
+version = "0.27.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4"
+
 [[package]]
 name = "glob"
 version = "0.3.1"
@ -2078,6 +2150,24 @@ dependencies = [
 "wasm-bindgen",
 ]

+[[package]]
+name = "inferno"
+version = "0.11.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fb7c1b80a1dfa604bb4a649a5c5aeef3d913f7c520cb42b40e534e8a61bcdfc"
+dependencies = [
+ "ahash 0.8.3",
+ "indexmap",
+ "is-terminal",
+ "itoa",
+ "log",
+ "num-format",
+ "once_cell",
+ "quick-xml",
+ "rgb",
+ "str_stack",
+]
+
 [[package]]
 name = "inout"
 version = "0.1.3"
@ -2350,6 +2440,15 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"

+[[package]]
+name = "memmap2"
+version = "0.5.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "memoffset"
 version = "0.8.0"
@ -2520,6 +2619,16 @@ dependencies = [
 "num-traits",
 ]

+[[package]]
+name = "num-format"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3"
+dependencies = [
+ "arrayvec",
+ "itoa",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.45"
@ -2550,6 +2659,15 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "object"
+version = "0.30.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.17.1"
@ -2908,6 +3026,28 @@ dependencies = [
 "plotters-backend",
 ]

+[[package]]
+name = "pprof"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "196ded5d4be535690899a4631cc9f18cdc41b7ebf24a79400f46f48e49a11059"
+dependencies = [
+ "backtrace",
+ "cfg-if",
+ "criterion",
+ "findshlibs",
+ "inferno",
+ "libc",
+ "log",
+ "nix",
+ "once_cell",
+ "parking_lot 0.12.1",
+ "smallvec",
+ "symbolic-demangle",
+ "tempfile",
+ "thiserror",
+]
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@ -3130,6 +3270,15 @@ version = "2.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"

+[[package]]
+name = "quick-xml"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.26"
@ -3377,6 +3526,15 @@ dependencies = [
 "web-sys",
 ]

+[[package]]
+name = "rgb"
+version = "0.8.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20ec2d3e3fc7a92ced357df9cebd5a10b6fb2aa1ee797bf7e9ce2f17dffc8f59"
+dependencies = [
+ "bytemuck",
+]
+
 [[package]]
 name = "ring"
 version = "0.16.20"
@ -3493,6 +3651,12 @@ dependencies = [
 "smallvec",
 ]

+[[package]]
+name = "rustc-demangle"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
+
 [[package]]
 name = "rustc-hash"
 version = "1.1.0"
@ -3902,6 +4066,12 @@ version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0"

+[[package]]
+name = "str_stack"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb"
+
 [[package]]
 name = "strsim"
 version = "0.10.0"
@ -3992,6 +4162,7 @@ dependencies = [
 "pbkdf2",
 "pharos",
 "pin-project-lite",
+ "pprof",
 "rand 0.8.5",
 "regex",
 "reqwest",
@ -4033,6 +4204,29 @@ dependencies = [
 "syn 1.0.109",
 ]

+[[package]]
+name = "symbolic-common"
+version = "10.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b55cdc318ede251d0957f07afe5fed912119b8c1bc5a7804151826db999e737"
+dependencies = [
+ "debugid",
+ "memmap2",
+ "stable_deref_trait",
+ "uuid",
+]
+
+[[package]]
+name = "symbolic-demangle"
+version = "10.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79be897be8a483a81fff6a3a4e195b4ac838ef73ca42d348b3f722da9902e489"
+dependencies = [
+ "cpp_demangle",
+ "rustc-demangle",
+ "symbolic-common",
+]
+
 [[package]]
 name = "syn"
 version = "1.0.109"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -25,6 +25,9 @@ opt-level = 3
 panic = 'abort'
 codegen-units = 1

+[profile.bench]
+strip = false
+
 [dependencies]
 argon2 = "0.5.0"
 base64 = "0.21.0"
--- a/lib/Cargo.toml
+++ b/lib/Cargo.toml
@ -105,6 +105,7 @@ url = "2.3.1"
 [dev-dependencies]
 criterion = "0.4"
 env_logger = "0.10.0"
+pprof = { version = "0.11.1", features = [ "flamegraph", "criterion" ] }
 temp-dir = "0.1.11"
 time = { version = "0.3.20", features = ["serde"] }
 tokio = { version = "1.27.0", features = ["macros", "sync", "rt-multi-thread"] }
--- a/lib/benches/README.md
+++ b/lib/benches/README.md
@ -8,5 +8,15 @@ establish the performance implications of a change.
 Execute the following command at the top level of the repository:

 ```console
-cargo bench --package surrealdb --no-default-features --features kv-mem
-```
+cargo bench --package surrealdb --no-default-features --features kv-mem,scripting,http
+```
+
+## Profiling
+
+Some of the benchmarks support CPU profiling:
+
+```console
+cargo bench --package surrealdb --no-default-features --features kv-mem,scripting,http -- --profile-time=5
+```
+
+Once complete, check the `target/criterion/**/profile/flamegraph.svg` files.
--- a/lib/benches/executor.rs
+++ b/lib/benches/executor.rs
@ -1,4 +1,5 @@
 use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput};
+use pprof::criterion::{Output, PProfProfiler};
 use surrealdb::{dbs::Session, kvs::Datastore};

 macro_rules! query {
@ -45,8 +46,20 @@ fn bench_executor(c: &mut Criterion) {
 		"CREATE person:one SET friend = person:two; CREATE person:two SET age = 30;",
 		"SELECT * FROM person:one.friend.age;"
 	);
+	#[cfg(feature = "scripting")]
+	query!(c, javascript_simple, "RETURN function() { return 1 + 1; };");
+	#[cfg(feature = "scripting")]
+	query!(
+		c,
+		javascript_function,
+		"RETURN function() { return surrealdb::functions::count([1, 2, 3]); };"
+	);
 	c.finish();
 }

-criterion_group!(benches, bench_executor);
+criterion_group!(
+	name = benches;
+	config = Criterion::default().with_profiler(PProfProfiler::new(1000, Output::Flamegraph(None)));
+	targets = bench_executor
+);
 criterion_main!(benches);
--- a/lib/benches/parser.rs
+++ b/lib/benches/parser.rs
@ -1,4 +1,5 @@
 use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput};
+use pprof::criterion::{Output, PProfProfiler};

 macro_rules! parser {
 	($c: expr, $name: ident, $parser: path, $text: expr) => {
@ -61,5 +62,9 @@ fn bench_parser(c: &mut Criterion) {
 	c.finish();
 }

-criterion_group!(benches, bench_parser);
+criterion_group!(
+	name = benches;
+	config = Criterion::default().with_profiler(PProfProfiler::new(1000, Output::Flamegraph(None)));
+	targets = bench_parser
+);
 criterion_main!(benches);