From e91011cc78b7ebd2dd96980876d5ee694baef33a Mon Sep 17 00:00:00 2001 From: Salvador Girones Gil Date: Thu, 3 Aug 2023 16:59:05 +0200 Subject: [PATCH] [rpc] Better tracing for WebSockets (#2325) --- .github/workflows/ci.yml | 26 +- Cargo.lock | 782 +++++++++++++---- Cargo.toml | 6 +- src/cli/start.rs | 2 +- src/cli/validator/parser/env_filter.rs | 18 +- src/net/mod.rs | 8 +- src/net/rpc.rs | 585 ++++++++----- src/net/signals.rs | 52 +- src/net/tracer.rs | 115 +-- src/rpc/mod.rs | 2 + src/rpc/res.rs | 72 +- src/telemetry/logs/mod.rs | 8 +- src/telemetry/mod.rs | 79 +- src/telemetry/traces/mod.rs | 7 +- src/telemetry/traces/otlp.rs | 27 +- src/telemetry/traces/rpc.rs | 31 + tests/cli_integration.rs | 73 +- tests/common/mod.rs | 214 ++++- tests/http_integration.rs | 51 +- tests/ws_integration.rs | 1109 ++++++++++++++++++++++++ 20 files changed, 2617 insertions(+), 650 deletions(-) create mode 100644 src/telemetry/traces/rpc.rs create mode 100644 tests/ws_integration.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e73427a..bda571f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,7 +133,7 @@ jobs: args: ci-clippy cli: - name: Test command line + name: CLI integration tests runs-on: ubuntu-latest steps: @@ -163,7 +163,7 @@ jobs: args: ci-cli-integration http-server: - name: Test HTTP server + name: HTTP integration tests runs-on: ubuntu-latest steps: @@ -192,6 +192,28 @@ jobs: command: make args: ci-http-integration + ws-server: + name: WebSocket integration tests + runs-on: ubuntu-latest + steps: + + - name: Install stable toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Checkout sources + uses: actions/checkout@v3 + + - name: Setup cache + uses: Swatinem/rust-cache@v2 + + - name: Install dependencies + run: | + sudo apt-get -y update + sudo apt-get -y install protobuf-compiler libprotobuf-dev + + - name: Run cargo test + run: cargo test --locked --no-default-features --features storage-mem --workspace --test ws_integration + test: name: Test workspace runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index fb0dc7b9..e25d7b75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9,12 +9,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8" dependencies = [ "bitflags 1.3.2", - "bytes", + "bytes 1.4.0", "futures-core", "futures-sink", "memchr", "pin-project-lite", - "tokio", + "tokio 1.29.1", "tokio-util", "tracing", ] @@ -43,7 +43,7 @@ dependencies = [ "base64 0.21.2", "bitflags 1.3.2", "brotli", - "bytes", + "bytes 1.4.0", "bytestring", "derive_more", "encoding_rs", @@ -61,8 +61,8 @@ dependencies = [ "pin-project-lite", "rand 0.8.5", "sha1", - "smallvec", - "tokio", + "smallvec 1.11.0", + "tokio 1.29.1", "tokio-util", "tracing", "zstd", @@ -98,7 +98,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e" dependencies = [ "futures-core", - "tokio", + "tokio 1.29.1", ] [[package]] @@ -112,10 +112,10 @@ dependencies = [ "actix-utils", "futures-core", "futures-util", - "mio", + "mio 0.8.8", "num_cpus", "socket2", - "tokio", + "tokio 1.29.1", "tracing", ] @@ -156,9 +156,9 @@ dependencies = [ "actix-utils", "actix-web-codegen", "ahash 0.7.6", - "bytes", + "bytes 1.4.0", "bytestring", - "cfg-if", + "cfg-if 1.0.0", "cookie", "derive_more", "encoding_rs", @@ -175,7 +175,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "smallvec", + "smallvec 1.11.0", "socket2", "time 0.3.23", "url", @@ -234,7 +234,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "getrandom 0.2.10", "once_cell", "version_check", @@ -523,9 +523,9 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" dependencies = [ - "futures", + "futures 0.3.28", "pharos", - "rustc_version", + "rustc_version 0.4.0", ] [[package]] @@ -551,7 +551,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi 0.1.19", "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -570,7 +570,7 @@ dependencies = [ "axum-core", "base64 0.21.2", "bitflags 1.3.2", - "bytes", + "bytes 1.4.0", "futures-util", "headers", "http", @@ -589,7 +589,7 @@ dependencies = [ "serde_urlencoded", "sha1", "sync_wrapper", - "tokio", + "tokio 1.29.1", "tokio-tungstenite 0.19.0", "tower", "tower-layer", @@ -615,7 +615,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", - "bytes", + "bytes 1.4.0", "futures-util", "http", "http-body", @@ -634,7 +634,7 @@ dependencies = [ "serde", "surrealdb", "thiserror", - "tokio", + "tokio 1.29.1", ] [[package]] @@ -646,7 +646,7 @@ dependencies = [ "axum", "axum-core", "axum-macros", - "bytes", + "bytes 1.4.0", "form_urlencoded", "futures-util", "http", @@ -656,7 +656,7 @@ dependencies = [ "pin-project-lite", "serde", "serde_html_form", - "tokio", + "tokio 1.29.1", "tower", "tower-http", "tower-layer", @@ -682,7 +682,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447f28c85900215cc1bea282f32d4a2f22d55c5a300afdfbc661c8d6a632e063" dependencies = [ "arc-swap", - "bytes", + "bytes 1.4.0", "futures-util", "http", "http-body", @@ -690,7 +690,7 @@ dependencies = [ "pin-project-lite", "rustls 0.21.5", "rustls-pemfile", - "tokio", + "tokio 1.29.1", "tokio-rustls 0.24.1", "tower-service", ] @@ -703,7 +703,7 @@ checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" dependencies = [ "addr2line", "cc", - "cfg-if", + "cfg-if 1.0.0", "libc", "miniz_oxide", "object", @@ -993,6 +993,16 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + [[package]] name = "bytes" version = "1.4.0" @@ -1005,7 +1015,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae" dependencies = [ - "bytes", + "bytes 1.4.0", ] [[package]] @@ -1101,6 +1111,12 @@ dependencies = [ "nom", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -1120,7 +1136,7 @@ dependencies = [ "serde", "time 0.1.45", "wasm-bindgen", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1244,7 +1260,16 @@ checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362" dependencies = [ "error-code", "str-buf", - "winapi", + "winapi 0.3.9", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags 1.3.2", ] [[package]] @@ -1259,7 +1284,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.8.16", ] [[package]] @@ -1268,7 +1293,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen", ] @@ -1311,7 +1336,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee34052ee3d93d6d8f3e6f81d85c47921f6653a19a7b70e939e3e602d893a674" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1329,7 +1354,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1344,7 +1369,7 @@ dependencies = [ "ciborium", "clap 3.2.25", "criterion-plot", - "futures", + "futures 0.3.28", "itertools", "lazy_static", "num-traits", @@ -1381,8 +1406,19 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.16", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" +dependencies = [ + "crossbeam-epoch 0.8.2", + "crossbeam-utils 0.7.2", + "maybe-uninit", ] [[package]] @@ -1391,9 +1427,24 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ - "cfg-if", - "crossbeam-epoch", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-epoch 0.9.15", + "crossbeam-utils 0.8.16", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "lazy_static", + "maybe-uninit", + "memoffset 0.5.6", + "scopeguard", ] [[package]] @@ -1403,19 +1454,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.16", "memoffset 0.9.0", "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "lazy_static", +] + [[package]] name = "crossbeam-utils" version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1475,11 +1548,11 @@ version = "5.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "hashbrown 0.14.0", - "lock_api", + "lock_api 0.4.10", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.8", ] [[package]] @@ -1498,7 +1571,7 @@ dependencies = [ "deadpool-runtime", "num_cpus", "retain_mut", - "tokio", + "tokio 1.29.1", ] [[package]] @@ -1547,7 +1620,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "rustc_version", + "rustc_version 0.4.0", "syn 1.0.109", ] @@ -1586,7 +1659,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "dirs-sys-next", ] @@ -1598,7 +1671,7 @@ checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", "redox_users", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1635,7 +1708,7 @@ dependencies = [ "arc-swap", "imbl", "thiserror", - "tokio", + "tokio 1.29.1", ] [[package]] @@ -1659,7 +1732,7 @@ version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1756,7 +1829,7 @@ version = "3.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "rustix 0.38.4", "windows-sys", ] @@ -1770,7 +1843,7 @@ dependencies = [ "cc", "lazy_static", "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1859,7 +1932,7 @@ dependencies = [ "foundationdb-gen", "foundationdb-macros", "foundationdb-sys", - "futures", + "futures 0.3.28", "memchr", "rand 0.8.5", "serde", @@ -1904,12 +1977,34 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags 1.3.2", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + [[package]] name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + [[package]] name = "futures" version = "0.3.28" @@ -2094,7 +2189,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -2105,7 +2200,7 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -2154,7 +2249,7 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ - "bytes", + "bytes 1.4.0", "fnv", "futures-core", "futures-sink", @@ -2162,7 +2257,7 @@ dependencies = [ "http", "indexmap 1.9.3", "slab", - "tokio", + "tokio 1.29.1", "tokio-util", "tracing", ] @@ -2214,7 +2309,7 @@ checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" dependencies = [ "base64 0.13.1", "bitflags 1.3.2", - "bytes", + "bytes 1.4.0", "headers-core", "http", "httpdate", @@ -2239,7 +2334,7 @@ checksum = "db04bc24a18b9ea980628ecf00e6c0264f3c1426dac36c00cb49b6fbad8b0743" dependencies = [ "atomic-polyfill", "hash32", - "rustc_version", + "rustc_version 0.4.0", "spin 0.9.8", "stable_deref_trait", ] @@ -2286,7 +2381,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes", + "bytes 1.4.0", "fnv", "itoa", ] @@ -2297,7 +2392,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes", + "bytes 1.4.0", "http", "pin-project-lite", ] @@ -2353,7 +2448,7 @@ version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ - "bytes", + "bytes 1.4.0", "futures-channel", "futures-core", "futures-util", @@ -2365,7 +2460,7 @@ dependencies = [ "itoa", "pin-project-lite", "socket2", - "tokio", + "tokio 1.29.1", "tower-service", "tracing", "want", @@ -2381,7 +2476,7 @@ dependencies = [ "http", "hyper", "rustls 0.21.5", - "tokio", + "tokio 1.29.1", "tokio-rustls 0.24.1", ] @@ -2393,7 +2488,7 @@ checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ "hyper", "pin-project-lite", - "tokio", + "tokio 1.29.1", "tokio-io-timeout", ] @@ -2403,10 +2498,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes", + "bytes 1.4.0", "hyper", "native-tls", - "tokio", + "tokio 1.29.1", "tokio-native-tls", ] @@ -2518,7 +2613,7 @@ dependencies = [ "js-sys", "rexie", "thiserror", - "tokio", + "tokio 1.29.1", "wasm-bindgen", ] @@ -2561,7 +2656,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -2575,6 +2670,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + [[package]] name = "ipnet" version = "2.8.0" @@ -2639,6 +2743,16 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + [[package]] name = "lalrpop" version = "0.19.12" @@ -2709,8 +2823,8 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" dependencies = [ - "cfg-if", - "winapi", + "cfg-if 1.0.0", + "winapi 0.3.9", ] [[package]] @@ -2792,6 +2906,15 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1" +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + [[package]] name = "lock_api" version = "0.4.10" @@ -2842,6 +2965,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + [[package]] name = "md-5" version = "0.10.5" @@ -2866,6 +2995,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.7.1" @@ -2915,6 +3053,25 @@ dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow", + "net2", + "slab", + "winapi 0.2.8", +] + [[package]] name = "mio" version = "0.8.8" @@ -2927,6 +3084,29 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio 0.6.23", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + [[package]] name = "nanoid" version = "0.4.0" @@ -2963,6 +3143,17 @@ dependencies = [ "tempfile", ] +[[package]] +name = "net2" +version = "0.2.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + [[package]] name = "new_debug_unreachable" version = "1.0.4" @@ -2975,7 +3166,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" dependencies = [ - "smallvec", + "smallvec 1.11.0", ] [[package]] @@ -2985,7 +3176,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.0", "libc", "memoffset 0.7.1", "pin-utils", @@ -3015,7 +3206,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" dependencies = [ "overload", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -3097,7 +3288,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" dependencies = [ "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", @@ -3151,14 +3342,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8af72d59a4484654ea8eb183fea5ae4eb6a41d7ac3e3bae5f4d2a282a3a7d3ca" dependencies = [ "async-trait", - "futures", + "futures 0.3.28", "futures-util", "http", "opentelemetry", "opentelemetry-proto", "prost", "thiserror", - "tokio", + "tokio 1.29.1", "tonic 0.8.3", ] @@ -3168,7 +3359,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "045f8eea8c0fa19f7d48e7bc3128a39c2e5c533d5c61298c548dfefc1064474c" dependencies = [ - "futures", + "futures 0.3.28", "futures-util", "opentelemetry", "prost", @@ -3209,7 +3400,7 @@ dependencies = [ "percent-encoding", "rand 0.8.5", "thiserror", - "tokio", + "tokio 1.29.1", "tokio-stream", ] @@ -3231,14 +3422,40 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api 0.3.4", + "parking_lot_core 0.6.3", + "rustc_version 0.2.3", +] + [[package]] name = "parking_lot" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ - "lock_api", - "parking_lot_core", + "lock_api 0.4.10", + "parking_lot_core 0.9.8", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66b810a62be75176a80873726630147a5ca780cd33921e0b5709033e66b0a" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall 0.1.57", + "rustc_version 0.2.3", + "smallvec 0.6.14", + "winapi 0.3.9", ] [[package]] @@ -3247,10 +3464,10 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall 0.3.5", - "smallvec", + "smallvec 1.11.0", "windows-targets", ] @@ -3320,8 +3537,8 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" dependencies = [ - "futures", - "rustc_version", + "futures 0.3.28", + "rustc_version 0.4.0", ] [[package]] @@ -3406,7 +3623,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "196ded5d4be535690899a4631cc9f18cdc41b7ebf24a79400f46f48e49a11059" dependencies = [ "backtrace", - "cfg-if", + "cfg-if 1.0.0", "criterion", "findshlibs", "inferno", @@ -3414,8 +3631,8 @@ dependencies = [ "log", "nix", "once_cell", - "parking_lot", - "smallvec", + "parking_lot 0.12.1", + "smallvec 1.11.0", "symbolic-demangle", "tempfile", "thiserror", @@ -3529,11 +3746,11 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fnv", "lazy_static", "memchr", - "parking_lot", + "parking_lot 0.12.1", "thiserror", ] @@ -3543,7 +3760,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ - "bytes", + "bytes 1.4.0", "prost-derive", ] @@ -3727,8 +3944,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils", + "crossbeam-deque 0.8.3", + "crossbeam-utils 0.8.16", "num_cpus", ] @@ -3744,6 +3961,12 @@ dependencies = [ "yasna", ] +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + [[package]] name = "redox_syscall" version = "0.2.16" @@ -3859,7 +4082,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ "base64 0.21.2", - "bytes", + "bytes 1.4.0", "encoding_rs", "futures-core", "futures-util", @@ -3883,7 +4106,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "tokio", + "tokio 1.29.1", "tokio-native-tls", "tokio-rustls 0.24.1", "tokio-util", @@ -3913,7 +4136,7 @@ dependencies = [ "js-sys", "num-traits", "thiserror", - "tokio", + "tokio 1.29.1", "wasm-bindgen", "web-sys", ] @@ -3939,7 +4162,7 @@ dependencies = [ "spin 0.5.2", "untrusted", "web-sys", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4077,7 +4300,7 @@ checksum = "1f39465655a1e3d8ae79c6d9e007f4953bfc5d55297602df9dc38f9ae9f1359a" dependencies = [ "heapless", "num-traits", - "smallvec", + "smallvec 1.11.0", ] [[package]] @@ -4100,7 +4323,7 @@ dependencies = [ "borsh", "bytecheck", "byteorder", - "bytes", + "bytes 1.4.0", "num-traits", "rand 0.8.5", "rkyv", @@ -4129,13 +4352,22 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver", + "semver 1.0.18", ] [[package]] @@ -4221,7 +4453,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dfc8644681285d1fb67a467fb3021bfea306b99b4146b166a1fe3ada965eece" dependencies = [ "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.0", "clipboard-win", "dirs-next", "fd-lock", @@ -4235,7 +4467,7 @@ dependencies = [ "unicode-segmentation", "unicode-width", "utf8parse", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4339,6 +4571,15 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + [[package]] name = "semver" version = "1.0.18" @@ -4348,6 +4589,12 @@ dependencies = [ "serde", ] +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "send_wrapper" version = "0.6.0" @@ -4486,10 +4733,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" dependencies = [ "dashmap", - "futures", + "futures 0.3.28", "lazy_static", "log", - "parking_lot", + "parking_lot 0.12.1", "serial_test_derive", ] @@ -4510,7 +4757,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest", ] @@ -4521,7 +4768,7 @@ version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest", ] @@ -4532,7 +4779,7 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest", ] @@ -4594,6 +4841,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" +dependencies = [ + "maybe-uninit", +] + [[package]] name = "smallvec" version = "1.11.0" @@ -4622,7 +4878,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4647,7 +4903,7 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" dependencies = [ - "lock_api", + "lock_api 0.4.10", ] [[package]] @@ -4663,10 +4919,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.0", "libc", "psm", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4707,7 +4963,7 @@ checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" dependencies = [ "new_debug_unreachable", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "phf_shared", "precomputed-hash", ] @@ -4735,9 +4991,10 @@ dependencies = [ "axum-extra", "axum-server", "base64 0.21.2", - "bytes", + "bytes 1.4.0", "clap 4.3.19", - "futures", + "env_logger", + "futures 0.3.28", "futures-util", "glob", "http", @@ -4762,14 +5019,17 @@ dependencies = [ "surrealdb", "temp-env", "tempfile", + "test-log", "thiserror", - "tokio", + "tokio 1.29.1", "tokio-stream", + "tokio-tungstenite 0.18.0", "tokio-util", "tonic 0.8.3", "tower", "tower-http", "tracing", + "tracing-futures", "tracing-opentelemetry", "tracing-subscriber", "urlencoding", @@ -4790,7 +5050,7 @@ dependencies = [ "bcrypt", "bincode", "bung", - "bytes", + "bytes 1.4.0", "cedar-policy", "chrono", "criterion", @@ -4801,7 +5061,7 @@ dependencies = [ "flume", "foundationdb", "fst", - "futures", + "futures 0.3.28", "futures-concurrency", "fuzzy-matcher", "geo", @@ -4830,7 +5090,7 @@ dependencies = [ "rust_decimal", "rustls 0.20.8", "scrypt", - "semver", + "semver 1.0.18", "serde", "serde_json", "serial_test", @@ -4845,7 +5105,7 @@ dependencies = [ "test-log", "thiserror", "time 0.3.23", - "tokio", + "tokio 1.29.1", "tokio-tungstenite 0.18.0", "tokio-util", "tracing", @@ -4880,7 +5140,7 @@ dependencies = [ "derive-new", "either", "fail", - "futures", + "futures 0.3.28", "lazy_static", "log", "pin-project", @@ -4888,11 +5148,11 @@ dependencies = [ "prost", "rand 0.8.5", "regex", - "semver", + "semver 1.0.18", "serde", "serde_derive", "thiserror", - "tokio", + "tokio 1.29.1", "tonic 0.9.2", ] @@ -4965,7 +5225,8 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9547444bfe52cbd79515c6c8087d8ae6ca8d64d2d31a27746320f5cb81d1a15c" dependencies = [ - "parking_lot", + "futures 0.3.28", + "parking_lot 0.12.1", ] [[package]] @@ -4974,7 +5235,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fastrand 2.0.0", "redox_syscall 0.3.5", "rustix 0.38.4", @@ -4989,7 +5250,7 @@ checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" dependencies = [ "dirs-next", "rustversion", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -5060,7 +5321,7 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "once_cell", ] @@ -5072,7 +5333,7 @@ checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -5136,6 +5397,30 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokio" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "mio 0.6.23", + "num_cpus", + "tokio-codec", + "tokio-current-thread", + "tokio-executor", + "tokio-fs", + "tokio-io", + "tokio-reactor", + "tokio-sync", + "tokio-tcp", + "tokio-threadpool", + "tokio-timer", + "tokio-udp", + "tokio-uds", +] + [[package]] name = "tokio" version = "1.29.1" @@ -5144,11 +5429,11 @@ checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ "autocfg", "backtrace", - "bytes", + "bytes 1.4.0", "libc", - "mio", + "mio 0.8.8", "num_cpus", - "parking_lot", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2", @@ -5156,6 +5441,59 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "tokio-codec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "tokio-io", +] + +[[package]] +name = "tokio-current-thread" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" +dependencies = [ + "futures 0.1.31", + "tokio-executor", +] + +[[package]] +name = "tokio-executor" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", +] + +[[package]] +name = "tokio-fs" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" +dependencies = [ + "futures 0.1.31", + "tokio-io", + "tokio-threadpool", +] + +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", +] + [[package]] name = "tokio-io-timeout" version = "1.2.0" @@ -5163,7 +5501,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" dependencies = [ "pin-project-lite", - "tokio", + "tokio 1.29.1", ] [[package]] @@ -5184,7 +5522,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio", + "tokio 1.29.1", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "lazy_static", + "log", + "mio 0.6.23", + "num_cpus", + "parking_lot 0.9.0", + "slab", + "tokio-executor", + "tokio-io", + "tokio-sync", ] [[package]] @@ -5194,7 +5551,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ "rustls 0.20.8", - "tokio", + "tokio 1.29.1", "webpki", ] @@ -5205,7 +5562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls 0.21.5", - "tokio", + "tokio 1.29.1", ] [[package]] @@ -5216,7 +5573,60 @@ checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", - "tokio", + "tokio 1.29.1", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures 0.1.31", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "mio 0.6.23", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" +dependencies = [ + "crossbeam-deque 0.7.4", + "crossbeam-queue", + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "lazy_static", + "log", + "num_cpus", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-timer" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "slab", + "tokio-executor", ] [[package]] @@ -5229,7 +5639,7 @@ dependencies = [ "log", "native-tls", "rustls 0.20.8", - "tokio", + "tokio 1.29.1", "tokio-native-tls", "tokio-rustls 0.23.4", "tungstenite 0.18.0", @@ -5245,22 +5655,55 @@ checksum = "ec509ac96e9a0c43427c74f003127d953a265737636129424288d27cb5c4b12c" dependencies = [ "futures-util", "log", - "tokio", + "tokio 1.29.1", "tungstenite 0.19.0", ] +[[package]] +name = "tokio-udp" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", + "mio 0.6.23", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-uds" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "libc", + "log", + "mio 0.6.23", + "mio-uds", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + [[package]] name = "tokio-util" version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ - "bytes", + "bytes 1.4.0", "futures-core", "futures-io", "futures-sink", "pin-project-lite", - "tokio", + "tokio 1.29.1", "tracing", ] @@ -5300,7 +5743,7 @@ dependencies = [ "async-trait", "axum", "base64 0.13.1", - "bytes", + "bytes 1.4.0", "futures-core", "futures-util", "h2", @@ -5312,7 +5755,7 @@ dependencies = [ "pin-project", "prost", "prost-derive", - "tokio", + "tokio 1.29.1", "tokio-stream", "tokio-util", "tower", @@ -5332,7 +5775,7 @@ dependencies = [ "async-trait", "axum", "base64 0.21.2", - "bytes", + "bytes 1.4.0", "futures-core", "futures-util", "h2", @@ -5344,7 +5787,7 @@ dependencies = [ "pin-project", "prost", "rustls-pemfile", - "tokio", + "tokio 1.29.1", "tokio-rustls 0.24.1", "tokio-stream", "tower", @@ -5366,7 +5809,7 @@ dependencies = [ "pin-project-lite", "rand 0.8.5", "slab", - "tokio", + "tokio 1.29.1", "tokio-util", "tower-layer", "tower-service", @@ -5381,7 +5824,7 @@ checksum = "55ae70283aba8d2a8b411c695c437fe25b8b5e44e23e780662002fc72fb47a82" dependencies = [ "base64 0.21.2", "bitflags 2.3.3", - "bytes", + "bytes 1.4.0", "futures-core", "futures-util", "http", @@ -5414,7 +5857,7 @@ version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-attributes", @@ -5449,6 +5892,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ "pin-project", + "tokio 0.1.22", "tracing", ] @@ -5488,7 +5932,7 @@ dependencies = [ "once_cell", "regex", "sharded-slab", - "smallvec", + "smallvec 1.11.0", "thread_local", "tracing", "tracing-core", @@ -5526,7 +5970,7 @@ checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" dependencies = [ "base64 0.13.1", "byteorder", - "bytes", + "bytes 1.4.0", "http", "httparse", "log", @@ -5547,7 +5991,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15fba1a6d6bb030745759a9a2a588bfe8490fc8b4751a277db3a0be1c9ebbf67" dependencies = [ "byteorder", - "bytes", + "bytes 1.4.0", "data-encoding", "http", "httparse", @@ -5754,7 +6198,7 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "serde", "serde_json", "wasm-bindgen-macro", @@ -5781,7 +6225,7 @@ version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -5835,9 +6279,9 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f656cd8858a5164932d8a90f936700860976ec21eb00e0fe2aa8cab13f6b4cf" dependencies = [ - "futures", + "futures 0.3.28", "js-sys", - "parking_lot", + "parking_lot 0.12.1", "pin-utils", "wasm-bindgen", ] @@ -5882,6 +6326,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + [[package]] name = "winapi" version = "0.3.9" @@ -5892,6 +6342,12 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -5904,7 +6360,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -6003,7 +6459,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -6016,7 +6472,7 @@ dependencies = [ "async-trait", "base64 0.21.2", "deadpool", - "futures", + "futures 0.3.28", "futures-timer", "http-types", "hyper", @@ -6025,7 +6481,17 @@ dependencies = [ "regex", "serde", "serde_json", - "tokio", + "tokio 1.29.1", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", ] [[package]] @@ -6035,11 +6501,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7999f5f4217fe3818726b66257a4475f71e74ffd190776ad053fa159e50737f5" dependencies = [ "async_io_stream", - "futures", + "futures 0.3.28", "js-sys", "log", "pharos", - "rustc_version", + "rustc_version 0.4.0", "send_wrapper", "thiserror", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index becb143f..f0787be2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ tokio-util = { version = "0.7.8", features = ["io"] } tower = "0.4.13" tower-http = { version = "0.4.2", features = ["trace", "sensitive-headers", "auth", "request-id", "util", "catch-panic", "cors", "set-header", "limit", "add-extension"] } tracing = "0.1" +tracing-futures = { version = "0.2.5", features = ["tokio"], default-features = false } tracing-opentelemetry = "0.19.0" tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } urlencoding = "2.1.2" @@ -77,11 +78,14 @@ nix = "0.26.2" [dev-dependencies] assert_fs = "1.0.13" +env_logger = "0.10.0" opentelemetry-proto = { version = "0.2.0", features = ["gen-tonic", "traces", "metrics", "logs"] } rcgen = "0.10.0" serial_test = "2.0.0" -temp-env = "0.3.4" +temp-env = { version = "0.3.4", features = ["async_closure"] } +test-log = { version = "0.2.12", features = ["trace"] } tokio-stream = { version = "0.1", features = ["net"] } +tokio-tungstenite = { version = "0.18.0" } tonic = "0.8.3" [package.metadata.deb] diff --git a/src/cli/start.rs b/src/cli/start.rs index 70deaa89..53febe25 100644 --- a/src/cli/start.rs +++ b/src/cli/start.rs @@ -115,7 +115,7 @@ pub async fn init( listen_addresses, dbs, web, - log: CustomEnvFilter(log), + log, tick_interval, no_banner, .. diff --git a/src/cli/validator/parser/env_filter.rs b/src/cli/validator/parser/env_filter.rs index 47240a38..6196419d 100644 --- a/src/cli/validator/parser/env_filter.rs +++ b/src/cli/validator/parser/env_filter.rs @@ -1,8 +1,9 @@ use clap::builder::{NonEmptyStringValueParser, PossibleValue, TypedValueParser}; use clap::error::{ContextKind, ContextValue, ErrorKind}; -use tracing::Level; use tracing_subscriber::EnvFilter; +use crate::telemetry::filter_from_value; + #[derive(Debug)] pub struct CustomEnvFilter(pub EnvFilter); @@ -37,20 +38,7 @@ impl TypedValueParser for CustomEnvFilterParser { let inner = NonEmptyStringValueParser::new(); let v = inner.parse_ref(cmd, arg, value)?; - let filter = (match v.as_str() { - // Don't show any logs at all - "none" => Ok(EnvFilter::default()), - // Check if we should show all log levels - "full" => Ok(EnvFilter::default().add_directive(Level::TRACE.into())), - // Otherwise, let's only show errors - "error" => Ok(EnvFilter::default().add_directive(Level::ERROR.into())), - // Specify the log level for each code area - "warn" | "info" | "debug" | "trace" => EnvFilter::builder() - .parse(format!("error,surreal={v},surrealdb={v},surrealdb::txn=error")), - // Let's try to parse the custom log level - _ => EnvFilter::builder().parse(v), - }) - .map_err(|e| { + let filter = filter_from_value(v.as_str()).map_err(|e| { let mut err = clap::Error::new(ErrorKind::ValueValidation).with_cmd(cmd); err.insert(ContextKind::Custom, ContextValue::String(e.to_string())); err.insert( diff --git a/src/net/mod.rs b/src/net/mod.rs index 2ef2f20d..8caa6e0a 100644 --- a/src/net/mod.rs +++ b/src/net/mod.rs @@ -131,7 +131,7 @@ pub async fn init() -> Result<(), Error> { // Setup the graceful shutdown with no timeout let handle = Handle::new(); - graceful_shutdown(handle.clone(), None); + let shutdown_handler = graceful_shutdown(handle.clone()); if let (Some(cert), Some(key)) = (&opt.crt, &opt.key) { // configure certificate and private key used by https @@ -156,6 +156,12 @@ pub async fn init() -> Result<(), Error> { .await?; }; + // Wait for the shutdown to finish + let _ = shutdown_handler.await; + + // Flush all telemetry data + opentelemetry::global::shutdown_tracer_provider(); + info!(target: LOG, "Web server stopped. Bye!"); Ok(()) diff --git a/src/net/rpc.rs b/src/net/rpc.rs index 86155f0e..9e7f6d71 100644 --- a/src/net/rpc.rs +++ b/src/net/rpc.rs @@ -7,26 +7,36 @@ use crate::err::Error; use crate::rpc::args::Take; use crate::rpc::paths::{ID, METHOD, PARAMS}; use crate::rpc::res; +use crate::rpc::res::Data; use crate::rpc::res::Failure; -use crate::rpc::res::Output; +use crate::rpc::res::IntoRpcResponse; +use crate::rpc::res::OutputFormat; +use crate::rpc::CONN_CLOSED_ERR; +use crate::telemetry::traces::rpc::span_for_request; use axum::routing::get; use axum::Extension; use axum::Router; use futures::{SinkExt, StreamExt}; +use futures_util::stream::SplitSink; +use futures_util::stream::SplitStream; use http_body::Body as HttpBody; use once_cell::sync::Lazy; use std::collections::BTreeMap; use std::collections::HashMap; use std::sync::Arc; use surrealdb::channel; -use surrealdb::channel::Sender; +use surrealdb::channel::{Receiver, Sender}; use surrealdb::dbs::{QueryType, Response, Session}; +use surrealdb::sql::serde::deserialize; use surrealdb::sql::Array; use surrealdb::sql::Object; use surrealdb::sql::Strand; use surrealdb::sql::Value; use tokio::sync::RwLock; -use tracing::instrument; +use tokio::task::JoinSet; +use tokio_util::sync::CancellationToken; +use tower_http::request_id::RequestId; +use tracing::Span; use uuid::Uuid; use axum::{ @@ -35,11 +45,12 @@ use axum::{ }; // Mapping of WebSocketID to WebSocket -type WebSockets = RwLock>>; +pub(crate) struct WebSocketRef(pub(crate) Sender, pub(crate) CancellationToken); +type WebSockets = RwLock>; // Mapping of LiveQueryID to WebSocketID type LiveQueries = RwLock>; -static WEBSOCKETS: Lazy = Lazy::new(WebSockets::default); +pub(super) static WEBSOCKETS: Lazy = Lazy::new(WebSockets::default); static LIVE_QUERIES: Lazy = Lazy::new(LiveQueries::default); pub(super) fn router() -> Router @@ -50,22 +61,36 @@ where Router::new().route("/rpc", get(handler)) } -async fn handler(ws: WebSocketUpgrade, Extension(sess): Extension) -> impl IntoResponse { +async fn handler( + ws: WebSocketUpgrade, + Extension(sess): Extension, + Extension(req_id): Extension, +) -> impl IntoResponse { // finalize the upgrade process by returning upgrade callback. // we can customize the callback by sending additional info such as address. - ws.on_upgrade(move |socket| handle_socket(socket, sess)) + ws.on_upgrade(move |socket| handle_socket(socket, sess, req_id)) } -async fn handle_socket(ws: WebSocket, sess: Session) { +async fn handle_socket(ws: WebSocket, sess: Session, req_id: RequestId) { let rpc = Rpc::new(sess); - Rpc::serve(rpc, ws).await + + // If the request ID is a valid UUID and is not already in use, use it as the WebSocket ID + match req_id.header_value().to_str().map(Uuid::parse_str) { + Ok(Ok(req_id)) if !WEBSOCKETS.read().await.contains_key(&req_id) => { + rpc.write().await.ws_id = req_id + } + _ => (), + } + + Rpc::serve(rpc, ws).await; } pub struct Rpc { session: Session, - format: Output, - uuid: Uuid, + format: OutputFormat, + ws_id: Uuid, vars: BTreeMap, + graceful_shutdown: CancellationToken, } impl Rpc { @@ -74,158 +99,247 @@ impl Rpc { // Create a new RPC variables store let vars = BTreeMap::new(); // Set the default output format - let format = Output::Json; - // Create a unique WebSocket id - let uuid = Uuid::new_v4(); - // Enable real-time live queries + let format = OutputFormat::Json; + // Enable real-time mode session.rt = true; // Create and store the Rpc connection Arc::new(RwLock::new(Rpc { session, format, - uuid, + ws_id: Uuid::new_v4(), vars, + graceful_shutdown: CancellationToken::new(), })) } /// Serve the RPC endpoint pub async fn serve(rpc: Arc>, ws: WebSocket) { - // Create a channel for sending messages - let (chn, mut rcv) = channel::new(MAX_CONCURRENT_CALLS); // Split the socket into send and recv - let (mut wtx, mut wrx) = ws.split(); - // Clone the channel for sending pings - let png = chn.clone(); - // The WebSocket has connected - Rpc::connected(rpc.clone(), chn.clone()).await; - // Send Ping messages to the client - tokio::task::spawn(async move { - // Create the interval ticker - let mut interval = tokio::time::interval(WEBSOCKET_PING_FREQUENCY); - // Loop indefinitely - loop { - // Wait for the timer - interval.tick().await; - // Create the ping message - let msg = Message::Ping(vec![]); - // Send the message to the client - if png.send(msg).await.is_err() { - // Exit out of the loop - break; - } - } - }); - // Send messages to the client - tokio::task::spawn(async move { - // Wait for the next message to send - while let Some(res) = rcv.next().await { - // Send the message to the client - if let Err(err) = wtx.send(res).await { - // Output the WebSocket error to the logs - trace!("WebSocket error: {:?}", err); - // It's already failed, so ignore error - let _ = wtx.close().await; - // Exit out of the loop - break; - } - } - }); - // Send notifications to the client - let moved_rpc = rpc.clone(); - tokio::task::spawn(async move { - let rpc = moved_rpc; - if let Some(channel) = DB.get().unwrap().notifications() { - while let Ok(notification) = channel.recv().await { - // Find which WebSocket the notification belongs to - if let Some(ws_id) = LIVE_QUERIES.read().await.get(¬ification.id) { - // Check to see if the WebSocket exists - if let Some(websocket) = WEBSOCKETS.read().await.get(ws_id) { - // Serialize the message to send - let message = res::success(None, notification); - // Get the current output format - let format = rpc.read().await.format.clone(); - // Send the notification to the client - message.send(format, websocket.clone()).await; - } - } - } - } - }); - // Get messages from the client - while let Some(msg) = wrx.next().await { - match msg { - // We've received a message from the client - // Ping is automatically handled by the WebSocket library - Ok(msg) => match msg { - Message::Text(_) => { - tokio::task::spawn(Rpc::call(rpc.clone(), msg, chn.clone())); - } - Message::Binary(_) => { - tokio::task::spawn(Rpc::call(rpc.clone(), msg, chn.clone())); - } - Message::Close(_) => { - break; - } - Message::Pong(_) => { - continue; - } - _ => { - // Ignore everything else - } - }, - // There was an error receiving the message - Err(err) => { - // Output the WebSocket error to the logs - trace!("WebSocket error: {:?}", err); - // Exit out of the loop - break; - } - } - } - // The WebSocket has disconnected - Rpc::disconnected(rpc.clone()).await; - } + let (sender, receiver) = ws.split(); + // Create an internal channel between the receiver and the sender + let (internal_sender, internal_receiver) = channel::new(MAX_CONCURRENT_CALLS); + + let ws_id = rpc.read().await.ws_id; - async fn connected(rpc: Arc>, chn: Sender) { - // Fetch the unique id of the WebSocket - let id = rpc.read().await.uuid; - // Log that the WebSocket has connected - trace!("WebSocket {} connected", id); // Store this WebSocket in the list of WebSockets - WEBSOCKETS.write().await.insert(id, chn); - } + WEBSOCKETS.write().await.insert( + ws_id, + WebSocketRef(internal_sender.clone(), rpc.read().await.graceful_shutdown.clone()), + ); + + trace!("WebSocket {} connected", ws_id); + + // Wait until all tasks finish + tokio::join!( + Self::ping(rpc.clone(), internal_sender.clone()), + Self::read(rpc.clone(), receiver, internal_sender.clone()), + Self::write(rpc.clone(), sender, internal_receiver.clone()), + Self::lq_notifications(rpc.clone()), + ); - async fn disconnected(rpc: Arc>) { - // Fetch the unique id of the WebSocket - let id = rpc.read().await.uuid; - // Log that the WebSocket has disconnected - trace!("WebSocket {} disconnected", id); - // Remove this WebSocket from the list of WebSockets - WEBSOCKETS.write().await.remove(&id); // Remove all live queries LIVE_QUERIES.write().await.retain(|key, value| { - if value == &id { + if value == &ws_id { trace!("Removing live query: {}", key); return false; } true }); + + // Remove this WebSocket from the list of WebSockets + WEBSOCKETS.write().await.remove(&ws_id); + + trace!("WebSocket {} disconnected", ws_id); } - /// Call RPC methods from the WebSocket - async fn call(rpc: Arc>, msg: Message, chn: Sender) { + /// Send Ping messages to the client + async fn ping(rpc: Arc>, internal_sender: Sender) { + // Create the interval ticker + let mut interval = tokio::time::interval(WEBSOCKET_PING_FREQUENCY); + let cancel_token = rpc.read().await.graceful_shutdown.clone(); + loop { + let is_shutdown = cancel_token.cancelled(); + tokio::select! { + _ = interval.tick() => { + let msg = Message::Ping(vec![]); + + // Send the message to the client and close the WebSocket connection if it fails + if internal_sender.send(msg).await.is_err() { + rpc.read().await.graceful_shutdown.cancel(); + break; + } + }, + _ = is_shutdown => break, + } + } + } + + /// Read messages sent from the client + async fn read( + rpc: Arc>, + mut receiver: SplitStream, + internal_sender: Sender, + ) { + // Collect all spawned tasks so we can wait for them at the end + let mut tasks = JoinSet::new(); + let cancel_token = rpc.read().await.graceful_shutdown.clone(); + loop { + let is_shutdown = cancel_token.cancelled(); + tokio::select! { + msg = receiver.next() => { + if let Some(msg) = msg { + match msg { + // We've received a message from the client + // Ping/Pong is automatically handled by the WebSocket library + Ok(msg) => match msg { + Message::Text(_) => { + tasks.spawn(Rpc::handle_msg(rpc.clone(), msg, internal_sender.clone())); + } + Message::Binary(_) => { + tasks.spawn(Rpc::handle_msg(rpc.clone(), msg, internal_sender.clone())); + } + Message::Close(_) => { + // Respond with a close message + if let Err(err) = internal_sender.send(Message::Close(None)).await { + trace!("WebSocket error when replying to the Close frame: {:?}", err); + }; + // Start the graceful shutdown of the WebSocket and close the channels + rpc.read().await.graceful_shutdown.cancel(); + let _ = internal_sender.close(); + break; + } + _ => { + // Ignore everything else + } + }, + Err(err) => { + trace!("WebSocket error: {:?}", err); + // Start the graceful shutdown of the WebSocket and close the channels + rpc.read().await.graceful_shutdown.cancel(); + let _ = internal_sender.close(); + // Exit out of the loop + break; + } + } + } + } + _ = is_shutdown => break, + } + } + + // Wait for all tasks to finish + while let Some(res) = tasks.join_next().await { + if let Err(err) = res { + error!("Error while handling RPC message: {}", err); + } + } + } + + /// Write messages to the client + async fn write( + rpc: Arc>, + mut sender: SplitSink, + mut internal_receiver: Receiver, + ) { + let cancel_token = rpc.read().await.graceful_shutdown.clone(); + loop { + let is_shutdown = cancel_token.cancelled(); + tokio::select! { + // Wait for the next message to send + msg = internal_receiver.next() => { + if let Some(res) = msg { + // Send the message to the client + if let Err(err) = sender.send(res).await { + if err.to_string() != CONN_CLOSED_ERR { + debug!("WebSocket error: {:?}", err); + } + // Close the WebSocket connection + rpc.read().await.graceful_shutdown.cancel(); + // Exit out of the loop + break; + } + } + }, + _ = is_shutdown => break, + } + } + } + + /// Send live query notifications to the client + async fn lq_notifications(rpc: Arc>) { + if let Some(channel) = DB.get().unwrap().notifications() { + let cancel_token = rpc.read().await.graceful_shutdown.clone(); + loop { + tokio::select! { + msg = channel.recv() => { + if let Ok(notification) = msg { + // Find which WebSocket the notification belongs to + if let Some(ws_id) = LIVE_QUERIES.read().await.get(¬ification.id) { + // Check to see if the WebSocket exists + if let Some(WebSocketRef(ws, _)) = WEBSOCKETS.read().await.get(ws_id) { + // Serialize the message to send + let message = res::success(None, notification); + // Get the current output format + let format = rpc.read().await.format.clone(); + // Send the notification to the client + message.send(format, ws.clone()).await + } + } + } + }, + _ = cancel_token.cancelled() => break, + } + } + } + } + + /// Handle individual WebSocket messages + async fn handle_msg(rpc: Arc>, msg: Message, chn: Sender) { // Get the current output format - let mut out = { rpc.read().await.format.clone() }; - // Clone the RPC - let rpc = rpc.clone(); + let mut out_fmt = rpc.read().await.format.clone(); + let span = span_for_request(&rpc.read().await.ws_id); + let _enter = span.enter(); // Parse the request + match Self::parse_request(msg).await { + Ok((id, method, params, _out_fmt)) => { + span.record( + "rpc.jsonrpc.request_id", + id.clone().map(|v| v.as_string()).unwrap_or(String::new()), + ); + if let Some(_out_fmt) = _out_fmt { + out_fmt = _out_fmt; + } + + // Process the request + let res = Self::process_request(rpc.clone(), &method, params).await; + + // Process the response + res.into_response(id).send(out_fmt, chn).await + } + Err(err) => { + // Process the response + res::failure(None, err).send(out_fmt, chn).await + } + } + } + + async fn parse_request( + msg: Message, + ) -> Result<(Option, String, Array, Option), Failure> { + let mut out_fmt = None; let req = match msg { // This is a binary message Message::Binary(val) => { // Use binary output - out = Output::Full; - // Deserialize the input - Value::from(val) + out_fmt = Some(OutputFormat::Full); + + match deserialize(&val) { + Ok(v) => v, + Err(_) => { + debug!("Error when trying to deserialize the request"); + return Err(Failure::PARSE_ERROR); + } + } } // This is a text message Message::Text(ref val) => { @@ -234,14 +348,15 @@ impl Rpc { // The SurrealQL message parsed ok Ok(v) => v, // The SurrealQL message failed to parse - _ => return res::failure(None, Failure::PARSE_ERROR).send(out, chn).await, + _ => return Err(Failure::PARSE_ERROR), } } // Unsupported message type - _ => return res::failure(None, Failure::INTERNAL_ERROR).send(out, chn).await, + _ => { + debug!("Unsupported message type: {:?}", msg); + return Err(res::Failure::custom("Unsupported message type")); + } }; - // Log the received request - trace!("RPC Received: {}", req); // Fetch the 'id' argument let id = match req.pick(&*ID) { v if v.is_none() => None, @@ -250,149 +365,180 @@ impl Rpc { v if v.is_number() => Some(v), v if v.is_strand() => Some(v), v if v.is_datetime() => Some(v), - _ => return res::failure(None, Failure::INVALID_REQUEST).send(out, chn).await, + _ => return Err(Failure::INVALID_REQUEST), }; // Fetch the 'method' argument let method = match req.pick(&*METHOD) { Value::Strand(v) => v.to_raw(), - _ => return res::failure(id, Failure::INVALID_REQUEST).send(out, chn).await, + _ => return Err(Failure::INVALID_REQUEST), }; + + // Now that we know the method, we can update the span + Span::current().record("rpc.method", &method); + Span::current().record("otel.name", format!("surrealdb.rpc/{}", method)); + // Fetch the 'params' argument let params = match req.pick(&*PARAMS) { Value::Array(v) => v, _ => Array::new(), }; + + Ok((id, method, params, out_fmt)) + } + + async fn process_request( + rpc: Arc>, + method: &str, + params: Array, + ) -> Result { + info!("Process RPC request"); + // Match the method to a function - let res = match &method[..] { - // Handle a ping message - "ping" => Ok(Value::None), + match method { + // Handle a surrealdb ping message + // + // This is used to keep the WebSocket connection alive in environments where the WebSocket protocol is not enough. + // For example, some browsers will wait for the TCP protocol to timeout before triggering an on_close event. This may take several seconds or even minutes in certain scenarios. + // By sending a ping message every few seconds from the client, we can force a connection check and trigger a an on_close event if the ping can't be sent. + // + "ping" => Ok(Value::None.into()), // Retrieve the current auth record "info" => match params.len() { - 0 => rpc.read().await.info().await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + 0 => rpc.read().await.info().await.map(Into::into).map_err(Into::into), + _ => Err(Failure::INVALID_PARAMS), }, // Switch to a specific namespace and database "use" => match params.needs_two() { - Ok((ns, db)) => rpc.write().await.yuse(ns, db).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok((ns, db)) => { + rpc.write().await.yuse(ns, db).await.map(Into::into).map_err(Into::into) + } + _ => Err(Failure::INVALID_PARAMS), }, // Signup to a specific authentication scope "signup" => match params.needs_one() { - Ok(Value::Object(v)) => rpc.write().await.signup(v).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok(Value::Object(v)) => { + rpc.write().await.signup(v).await.map(Into::into).map_err(Into::into) + } + _ => Err(Failure::INVALID_PARAMS), }, // Signin as a root, namespace, database or scope user "signin" => match params.needs_one() { - Ok(Value::Object(v)) => rpc.write().await.signin(v).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok(Value::Object(v)) => { + rpc.write().await.signin(v).await.map(Into::into).map_err(Into::into) + } + _ => Err(Failure::INVALID_PARAMS), }, // Invalidate the current authentication session "invalidate" => match params.len() { - 0 => rpc.write().await.invalidate().await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + 0 => rpc.write().await.invalidate().await.map(Into::into).map_err(Into::into), + _ => Err(Failure::INVALID_PARAMS), }, // Authenticate using an authentication token "authenticate" => match params.needs_one() { - Ok(Value::Strand(v)) => rpc.write().await.authenticate(v).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok(Value::Strand(v)) => { + rpc.write().await.authenticate(v).await.map(Into::into).map_err(Into::into) + } + _ => Err(Failure::INVALID_PARAMS), }, // Kill a live query using a query id "kill" => match params.needs_one() { - Ok(v) if v.is_uuid() => rpc.read().await.kill(v).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok(v) if v.is_uuid() => { + rpc.read().await.kill(v).await.map(Into::into).map_err(Into::into) + } + _ => Err(Failure::INVALID_PARAMS), }, // Setup a live query on a specific table "live" => match params.needs_one_or_two() { - Ok((v, d)) if v.is_table() => rpc.read().await.live(v, d).await, - Ok((v, d)) if v.is_strand() => rpc.read().await.live(v, d).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok((v, d)) if v.is_table() => { + rpc.read().await.live(v, d).await.map(Into::into).map_err(Into::into) + } + Ok((v, d)) if v.is_strand() => { + rpc.read().await.live(v, d).await.map(Into::into).map_err(Into::into) + } + _ => Err(Failure::INVALID_PARAMS), }, // Specify a connection-wide parameter - "let" => match params.needs_one_or_two() { - Ok((Value::Strand(s), v)) => rpc.write().await.set(s, v).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, - }, - // Specify a connection-wide parameter - "set" => match params.needs_one_or_two() { - Ok((Value::Strand(s), v)) => rpc.write().await.set(s, v).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + "let" | "set" => match params.needs_one_or_two() { + Ok((Value::Strand(s), v)) => { + rpc.write().await.set(s, v).await.map(Into::into).map_err(Into::into) + } + _ => Err(Failure::INVALID_PARAMS), }, // Unset and clear a connection-wide parameter "unset" => match params.needs_one() { - Ok(Value::Strand(s)) => rpc.write().await.unset(s).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok(Value::Strand(s)) => { + rpc.write().await.unset(s).await.map(Into::into).map_err(Into::into) + } + _ => Err(Failure::INVALID_PARAMS), }, // Select a value or values from the database "select" => match params.needs_one() { - Ok(v) => rpc.read().await.select(v).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok(v) => rpc.read().await.select(v).await.map(Into::into).map_err(Into::into), + _ => Err(Failure::INVALID_PARAMS), }, // Insert a value or values in the database "insert" => match params.needs_one_or_two() { - Ok((v, o)) => rpc.read().await.insert(v, o).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok((v, o)) => { + rpc.read().await.insert(v, o).await.map(Into::into).map_err(Into::into) + } + _ => Err(Failure::INVALID_PARAMS), }, // Create a value or values in the database "create" => match params.needs_one_or_two() { - Ok((v, o)) => rpc.read().await.create(v, o).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok((v, o)) => { + rpc.read().await.create(v, o).await.map(Into::into).map_err(Into::into) + } + _ => Err(Failure::INVALID_PARAMS), }, // Update a value or values in the database using `CONTENT` "update" => match params.needs_one_or_two() { - Ok((v, o)) => rpc.read().await.update(v, o).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok((v, o)) => { + rpc.read().await.update(v, o).await.map(Into::into).map_err(Into::into) + } + _ => Err(Failure::INVALID_PARAMS), }, // Update a value or values in the database using `MERGE` "change" | "merge" => match params.needs_one_or_two() { - Ok((v, o)) => rpc.read().await.change(v, o).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok((v, o)) => { + rpc.read().await.change(v, o).await.map(Into::into).map_err(Into::into) + } + _ => Err(Failure::INVALID_PARAMS), }, // Update a value or values in the database using `PATCH` "modify" | "patch" => match params.needs_one_or_two() { - Ok((v, o)) => rpc.read().await.modify(v, o).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok((v, o)) => { + rpc.read().await.modify(v, o).await.map(Into::into).map_err(Into::into) + } + _ => Err(Failure::INVALID_PARAMS), }, // Delete a value or values from the database "delete" => match params.needs_one() { - Ok(v) => rpc.read().await.delete(v).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok(v) => rpc.read().await.delete(v).await.map(Into::into).map_err(Into::into), + _ => Err(Failure::INVALID_PARAMS), }, // Specify the output format for text requests "format" => match params.needs_one() { - Ok(Value::Strand(v)) => rpc.write().await.format(v).await, - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + Ok(Value::Strand(v)) => { + rpc.write().await.format(v).await.map(Into::into).map_err(Into::into) + } + _ => Err(Failure::INVALID_PARAMS), }, // Get the current server version "version" => match params.len() { 0 => Ok(format!("{PKG_NAME}-{}", *PKG_VERSION).into()), - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + _ => Err(Failure::INVALID_PARAMS), }, // Run a full SurrealQL query against the database "query" => match params.needs_one_or_two() { Ok((Value::Strand(s), o)) if o.is_none_or_null() => { - return match rpc.read().await.query(s).await { - Ok(v) => res::success(id, v).send(out, chn).await, - Err(e) => { - res::failure(id, Failure::custom(e.to_string())).send(out, chn).await - } - }; + rpc.read().await.query(s).await.map(Into::into).map_err(Into::into) } Ok((Value::Strand(s), Value::Object(o))) => { - return match rpc.read().await.query_with(s, o).await { - Ok(v) => res::success(id, v).send(out, chn).await, - Err(e) => { - res::failure(id, Failure::custom(e.to_string())).send(out, chn).await - } - }; + rpc.read().await.query_with(s, o).await.map(Into::into).map_err(Into::into) } - _ => return res::failure(id, Failure::INVALID_PARAMS).send(out, chn).await, + _ => Err(Failure::INVALID_PARAMS), }, - _ => return res::failure(id, Failure::METHOD_NOT_FOUND).send(out, chn).await, - }; - // Return the final response - match res { - Ok(v) => res::success(id, v).send(out, chn).await, - Err(e) => res::failure(id, Failure::custom(e.to_string())).send(out, chn).await, + _ => Err(Failure::METHOD_NOT_FOUND), } } @@ -402,15 +548,14 @@ impl Rpc { async fn format(&mut self, out: Strand) -> Result { match out.as_str() { - "json" | "application/json" => self.format = Output::Json, - "cbor" | "application/cbor" => self.format = Output::Cbor, - "pack" | "application/pack" => self.format = Output::Pack, + "json" | "application/json" => self.format = OutputFormat::Json, + "cbor" | "application/cbor" => self.format = OutputFormat::Cbor, + "pack" | "application/pack" => self.format = OutputFormat::Pack, _ => return Err(Error::InvalidType), }; Ok(Value::None) } - #[instrument(skip_all, name = "rpc use", fields(websocket=self.uuid.to_string()))] async fn yuse(&mut self, ns: Value, db: Value) -> Result { if let Value::Strand(ns) = ns { self.session.ns = Some(ns.0); @@ -421,7 +566,6 @@ impl Rpc { Ok(Value::None) } - #[instrument(skip_all, name = "rpc signup", fields(websocket=self.uuid.to_string()))] async fn signup(&mut self, vars: Object) -> Result { let kvs = DB.get().unwrap(); surrealdb::iam::signup::signup(kvs, &mut self.session, vars) @@ -430,7 +574,6 @@ impl Rpc { .map_err(Into::into) } - #[instrument(skip_all, name = "rpc signin", fields(websocket=self.uuid.to_string()))] async fn signin(&mut self, vars: Object) -> Result { let kvs = DB.get().unwrap(); surrealdb::iam::signin::signin(kvs, &mut self.session, vars) @@ -438,13 +581,11 @@ impl Rpc { .map(Into::into) .map_err(Into::into) } - #[instrument(skip_all, name = "rpc invalidate", fields(websocket=self.uuid.to_string()))] async fn invalidate(&mut self) -> Result { surrealdb::iam::clear::clear(&mut self.session)?; Ok(Value::None) } - #[instrument(skip_all, name = "rpc auth", fields(websocket=self.uuid.to_string()))] async fn authenticate(&mut self, token: Strand) -> Result { let kvs = DB.get().unwrap(); surrealdb::iam::verify::token(kvs, &mut self.session, &token.0).await?; @@ -455,7 +596,6 @@ impl Rpc { // Methods for identification // ------------------------------ - #[instrument(skip_all, name = "rpc info", fields(websocket=self.uuid.to_string()))] async fn info(&self) -> Result { // Get a database reference let kvs = DB.get().unwrap(); @@ -473,7 +613,6 @@ impl Rpc { // Methods for setting variables // ------------------------------ - #[instrument(skip_all, name = "rpc set", fields(websocket=self.uuid.to_string()))] async fn set(&mut self, key: Strand, val: Value) -> Result { match val { // Remove the variable if undefined @@ -484,7 +623,6 @@ impl Rpc { Ok(Value::Null) } - #[instrument(skip_all, name = "rpc unset", fields(websocket=self.uuid.to_string()))] async fn unset(&mut self, key: Strand) -> Result { self.vars.remove(&key.0); Ok(Value::Null) @@ -494,7 +632,6 @@ impl Rpc { // Methods for live queries // ------------------------------ - #[instrument(skip_all, name = "rpc kill", fields(websocket=self.uuid.to_string()))] async fn kill(&self, id: Value) -> Result { // Specify the SQL query string let sql = "KILL $id"; @@ -513,7 +650,6 @@ impl Rpc { } } - #[instrument(skip_all, name = "rpc live", fields(websocket=self.uuid.to_string()))] async fn live(&self, tb: Value, diff: Value) -> Result { // Specify the SQL query string let sql = match diff.is_true() { @@ -539,7 +675,6 @@ impl Rpc { // Methods for selecting // ------------------------------ - #[instrument(skip_all, name = "rpc select", fields(websocket=self.uuid.to_string()))] async fn select(&self, what: Value) -> Result { // Return a single result? let one = what.is_thing(); @@ -567,7 +702,6 @@ impl Rpc { // Methods for inserting // ------------------------------ - #[instrument(skip_all, name = "rpc insert", fields(websocket=self.uuid.to_string()))] async fn insert(&self, what: Value, data: Value) -> Result { // Return a single result? let one = what.is_thing(); @@ -596,7 +730,6 @@ impl Rpc { // Methods for creating // ------------------------------ - #[instrument(skip_all, name = "rpc create", fields(websocket=self.uuid.to_string()))] async fn create(&self, what: Value, data: Value) -> Result { // Return a single result? let one = what.is_thing(); @@ -625,7 +758,6 @@ impl Rpc { // Methods for updating // ------------------------------ - #[instrument(skip_all, name = "rpc update", fields(websocket=self.uuid.to_string()))] async fn update(&self, what: Value, data: Value) -> Result { // Return a single result? let one = what.is_thing(); @@ -654,7 +786,6 @@ impl Rpc { // Methods for changing // ------------------------------ - #[instrument(skip_all, name = "rpc change", fields(websocket=self.uuid.to_string()))] async fn change(&self, what: Value, data: Value) -> Result { // Return a single result? let one = what.is_thing(); @@ -683,7 +814,6 @@ impl Rpc { // Methods for modifying // ------------------------------ - #[instrument(skip_all, name = "rpc modify", fields(websocket=self.uuid.to_string()))] async fn modify(&self, what: Value, data: Value) -> Result { // Return a single result? let one = what.is_thing(); @@ -712,7 +842,6 @@ impl Rpc { // Methods for deleting // ------------------------------ - #[instrument(skip_all, name = "rpc delete", fields(websocket=self.uuid.to_string()))] async fn delete(&self, what: Value) -> Result { // Return a single result? let one = what.is_thing(); @@ -740,7 +869,6 @@ impl Rpc { // Methods for querying // ------------------------------ - #[instrument(skip_all, name = "rpc query", fields(websocket=self.uuid.to_string()))] async fn query(&self, sql: Strand) -> Result, Error> { // Get a database reference let kvs = DB.get().unwrap(); @@ -756,7 +884,6 @@ impl Rpc { Ok(res) } - #[instrument(skip_all, name = "rpc query_with", fields(websocket=self.uuid.to_string()))] async fn query_with(&self, sql: Strand, mut vars: Object) -> Result, Error> { // Get a database reference let kvs = DB.get().unwrap(); @@ -781,8 +908,8 @@ impl Rpc { QueryType::Live => { if let Ok(Value::Uuid(lqid)) = &res.result { // Match on Uuid type - LIVE_QUERIES.write().await.insert(lqid.0, self.uuid); - trace!("Registered live query {} on websocket {}", lqid, self.uuid); + LIVE_QUERIES.write().await.insert(lqid.0, self.ws_id); + trace!("Registered live query {} on websocket {}", lqid, self.ws_id); } } QueryType::Kill => { diff --git a/src/net/signals.rs b/src/net/signals.rs index 1c7084a1..a14f9b96 100644 --- a/src/net/signals.rs +++ b/src/net/signals.rs @@ -1,17 +1,57 @@ use std::time::Duration; use axum_server::Handle; +use tokio::task::JoinHandle; -use crate::err::Error; +use crate::{ + err::Error, + net::rpc::{WebSocketRef, WEBSOCKETS}, +}; -/// Start a graceful shutdown on the Axum Handle when a shutdown signal is received. -pub fn graceful_shutdown(handle: Handle, dur: Option) { +/// Start a graceful shutdown: +/// * Signal the Axum Handle when a shutdown signal is received. +/// * Stop all WebSocket connections. +/// +/// A second signal will force an immediate shutdown. +pub fn graceful_shutdown(http_handle: Handle) -> JoinHandle<()> { tokio::spawn(async move { let result = listen().await.expect("Failed to listen to shutdown signal"); - info!(target: super::LOG, "{} received. Start graceful shutdown...", result); + info!(target: super::LOG, "{} received. Waiting for graceful shutdown... A second signal will force an immediate shutdown", result); - handle.graceful_shutdown(dur) - }); + tokio::select! { + // Start a normal graceful shutdown + _ = async { + // First stop accepting new HTTP requests + http_handle.graceful_shutdown(None); + + // Close all WebSocket connections. Queued messages will still be processed. + for (_, WebSocketRef(_, cancel_token)) in WEBSOCKETS.read().await.iter() { + cancel_token.cancel(); + }; + + // Wait for all existing WebSocket connections to gracefully close + while WEBSOCKETS.read().await.len() > 0 { + tokio::time::sleep(Duration::from_millis(100)).await; + }; + } => (), + // Force an immediate shutdown if a second signal is received + _ = async { + if let Ok(signal) = listen().await { + warn!(target: super::LOG, "{} received during graceful shutdown. Terminate immediately...", signal); + } else { + error!(target: super::LOG, "Failed to listen to shutdown signal. Terminate immediately..."); + } + + // Force an immediate shutdown + http_handle.shutdown(); + + // Close all WebSocket connections immediately + if let Ok(mut writer) = WEBSOCKETS.try_write() { + writer.drain(); + } + } => (), + } + }) } #[cfg(unix)] diff --git a/src/net/tracer.rs b/src/net/tracer.rs index fa210191..f0db0400 100644 --- a/src/net/tracer.rs +++ b/src/net/tracer.rs @@ -1,119 +1,15 @@ use std::{fmt, time::Duration}; -use axum::{ - body::{boxed, Body, BoxBody}, - extract::MatchedPath, - headers::{ - authorization::{Basic, Bearer}, - Authorization, Origin, - }, - Extension, RequestPartsExt, TypedHeader, -}; -use futures_util::future::BoxFuture; -use http::{header, request::Parts, StatusCode}; +use axum::extract::MatchedPath; +use http::header; use hyper::{Request, Response}; -use surrealdb::{ - dbs::Session, - iam::verify::{basic, token}, -}; use tower_http::{ - auth::AsyncAuthorizeRequest, request_id::RequestId, trace::{MakeSpan, OnFailure, OnRequest, OnResponse}, }; use tracing::{field, Level, Span}; -use crate::{dbs::DB, err::Error}; - -use super::{client_ip::ExtractClientIP, AppState}; - -/// -/// SurrealAuth is a tower layer that implements the AsyncAuthorizeRequest trait. -/// It is used to authorize requests to SurrealDB using Basic or Token authentication. -/// -/// It has to be used in conjunction with the tower_http::auth::RequireAuthorizationLayer layer: -/// -/// ```rust -/// use tower_http::auth::RequireAuthorizationLayer; -/// use surrealdb::net::SurrealAuth; -/// use axum::Router; -/// -/// let auth = RequireAuthorizationLayer::new(SurrealAuth); -/// -/// let app = Router::new() -/// .route("/version", get(|| async { "0.1.0" })) -/// .layer(auth); -/// ``` -#[derive(Clone, Copy)] -pub(super) struct SurrealAuth; - -impl AsyncAuthorizeRequest for SurrealAuth -where - B: Send + Sync + 'static, -{ - type RequestBody = B; - type ResponseBody = BoxBody; - type Future = BoxFuture<'static, Result, Response>>; - - fn authorize(&mut self, request: Request) -> Self::Future { - Box::pin(async { - let (mut parts, body) = request.into_parts(); - match check_auth(&mut parts).await { - Ok(sess) => { - parts.extensions.insert(sess); - Ok(Request::from_parts(parts, body)) - } - Err(err) => { - let unauthorized_response = Response::builder() - .status(StatusCode::UNAUTHORIZED) - .body(boxed(Body::from(err.to_string()))) - .unwrap(); - Err(unauthorized_response) - } - } - }) - } -} - -async fn check_auth(parts: &mut Parts) -> Result { - let kvs = DB.get().unwrap(); - - let or = if let Ok(or) = parts.extract::>().await { - if !or.is_null() { - Some(or.to_string()) - } else { - None - } - } else { - None - }; - - let id = parts.headers.get("id").map(|v| v.to_str().unwrap().to_string()); // TODO: Use a TypedHeader - let ns = parts.headers.get("ns").map(|v| v.to_str().unwrap().to_string()); // TODO: Use a TypedHeader - let db = parts.headers.get("db").map(|v| v.to_str().unwrap().to_string()); // TODO: Use a TypedHeader - - let Extension(state) = parts.extract::>().await.map_err(|err| { - tracing::error!("Error extracting the app state: {:?}", err); - Error::InvalidAuth - })?; - let ExtractClientIP(ip) = - parts.extract_with_state(&state).await.unwrap_or(ExtractClientIP(None)); - - // Create session - #[rustfmt::skip] - let mut session = Session { ip, or, id, ns, db, ..Default::default() }; - - // If Basic authentication data was supplied - if let Ok(au) = parts.extract::>>().await { - basic(kvs, &mut session, au.username(), au.password()).await.map_err(|e| e.into()) - } else if let Ok(au) = parts.extract::>>().await { - token(kvs, &mut session, au.token()).await.map_err(|e| e.into()) - } else { - Err(Error::InvalidAuth) - }?; - - Ok(session) -} +use super::client_ip::ExtractClientIP; /// /// HttpTraceLayerHooks implements custom hooks for the tower_http::trace::TraceLayer layer. @@ -139,7 +35,6 @@ impl MakeSpan for HttpTraceLayerHooks { fn make_span(&mut self, req: &Request) -> Span { // The fields follow the OTEL semantic conventions: https://github.com/open-telemetry/opentelemetry-specification/blob/v1.23.0/specification/trace/semantic_conventions/http.md let span = tracing::info_span!( - target: "surreal::http", "request", otel.name = field::Empty, otel.kind = "server", @@ -154,10 +49,10 @@ impl MakeSpan for HttpTraceLayerHooks { network.protocol.name = "http", network.protocol.version = format!("{:?}", req.version()).strip_prefix("HTTP/"), client.address = field::Empty, - client.port = field::Empty, + client.port = field::Empty, client.socket.address = field::Empty, server.address = field::Empty, - server.port = field::Empty, + server.port = field::Empty, // set on the response hook http.latency.ms = field::Empty, http.response.status_code = field::Empty, diff --git a/src/rpc/mod.rs b/src/rpc/mod.rs index 9be3b132..c11136eb 100644 --- a/src/rpc/mod.rs +++ b/src/rpc/mod.rs @@ -1,3 +1,5 @@ pub mod args; pub mod paths; pub mod res; + +pub(crate) static CONN_CLOSED_ERR: &str = "Connection closed normally"; diff --git a/src/rpc/res.rs b/src/rpc/res.rs index 6f8c2f51..ebf1b047 100644 --- a/src/rpc/res.rs +++ b/src/rpc/res.rs @@ -7,10 +7,13 @@ use surrealdb::dbs; use surrealdb::dbs::Notification; use surrealdb::sql; use surrealdb::sql::Value; -use tracing::instrument; +use tracing::Span; -#[derive(Clone)] -pub enum Output { +use crate::err; +use crate::rpc::CONN_CLOSED_ERR; + +#[derive(Debug, Clone)] +pub enum OutputFormat { Json, // JSON Cbor, // CBOR Pack, // MessagePack @@ -37,6 +40,12 @@ impl From for Data { } } +impl From for Data { + fn from(v: String) -> Self { + Data::Other(Value::from(v)) + } +} + impl From> for Data { fn from(v: Vec) -> Self { Data::Query(v) @@ -82,28 +91,45 @@ impl Response { } /// Send the response to the WebSocket channel - #[instrument(skip_all, name = "rpc response", fields(response = ?self))] - pub async fn send(self, out: Output, chn: Sender) { + pub async fn send(self, out: OutputFormat, chn: Sender) { + let span = Span::current(); + + info!("Process RPC response"); + + if let Err(err) = &self.result { + span.record("otel.status_code", "Error"); + span.record( + "otel.status_message", + format!("code: {}, message: {}", err.code, err.message), + ); + span.record("rpc.jsonrpc.error_code", err.code); + span.record("rpc.jsonrpc.error_message", err.message.as_ref()); + } + let message = match out { - Output::Json => { + OutputFormat::Json => { let res = serde_json::to_string(&self.simplify()).unwrap(); Message::Text(res) } - Output::Cbor => { + OutputFormat::Cbor => { let res = serde_cbor::to_vec(&self.simplify()).unwrap(); Message::Binary(res) } - Output::Pack => { + OutputFormat::Pack => { let res = serde_pack::to_vec(&self.simplify()).unwrap(); Message::Binary(res) } - Output::Full => { + OutputFormat::Full => { let res = surrealdb::sql::serde::serialize(&self).unwrap(); Message::Binary(res) } }; - let _ = chn.send(message).await; - trace!("Response sent"); + + if let Err(err) = chn.send(message).await { + if err.to_string() != CONN_CLOSED_ERR { + error!("Error sending response: {}", err); + } + }; } } @@ -113,6 +139,7 @@ pub struct Failure { message: Cow<'static, str>, } +#[allow(dead_code)] impl Failure { pub const PARSE_ERROR: Failure = Failure { code: -32700, @@ -165,3 +192,26 @@ pub fn failure(id: Option, err: Failure) -> Response { result: Err(err), } } + +impl From for Failure { + fn from(err: err::Error) -> Self { + Failure::custom(err.to_string()) + } +} + +pub trait IntoRpcResponse { + fn into_response(self, id: Option) -> Response; +} + +impl IntoRpcResponse for Result +where + T: Into, + E: Into, +{ + fn into_response(self, id: Option) -> Response { + match self { + Ok(v) => success(id, v.into()), + Err(err) => failure(id, err.into()), + } + } +} diff --git a/src/telemetry/logs/mod.rs b/src/telemetry/logs/mod.rs index 4b60f159..95eeb027 100644 --- a/src/telemetry/logs/mod.rs +++ b/src/telemetry/logs/mod.rs @@ -1,8 +1,10 @@ use tracing::Subscriber; use tracing_subscriber::fmt::format::FmtSpan; -use tracing_subscriber::{EnvFilter, Layer}; +use tracing_subscriber::Layer; -pub fn new(level: String) -> Box + Send + Sync> +use crate::cli::validator::parser::env_filter::CustomEnvFilter; + +pub fn new(filter: CustomEnvFilter) -> Box + Send + Sync> where S: Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a> + Send + Sync, { @@ -11,6 +13,6 @@ where .with_ansi(true) .with_span_events(FmtSpan::NONE) .with_writer(std::io::stderr) - .with_filter(EnvFilter::builder().parse(level).unwrap()) + .with_filter(filter.0) .boxed() } diff --git a/src/telemetry/mod.rs b/src/telemetry/mod.rs index c87490ea..b1be2d1e 100644 --- a/src/telemetry/mod.rs +++ b/src/telemetry/mod.rs @@ -1,6 +1,6 @@ mod logs; pub mod metrics; -mod traces; +pub mod traces; use std::time::Duration; @@ -11,8 +11,7 @@ use opentelemetry::sdk::resource::{ }; use opentelemetry::sdk::Resource; use opentelemetry::KeyValue; -use tracing::Subscriber; -use tracing_subscriber::fmt::format::FmtSpan; +use tracing::{Level, Subscriber}; use tracing_subscriber::prelude::*; use tracing_subscriber::util::SubscriberInitExt; #[cfg(feature = "has-storage")] @@ -39,53 +38,75 @@ pub static OTEL_DEFAULT_RESOURCE: Lazy = Lazy::new(|| { } }); -#[derive(Default, Debug, Clone)] +#[derive(Debug, Clone)] pub struct Builder { - log_level: Option, - filter: Option, + filter: CustomEnvFilter, } pub fn builder() -> Builder { Builder::default() } +impl Default for Builder { + fn default() -> Self { + Self { + filter: CustomEnvFilter(EnvFilter::default()), + } + } +} + impl Builder { /// Set the log level on the builder pub fn with_log_level(mut self, log_level: &str) -> Self { - self.log_level = Some(log_level.to_string()); + if let Ok(filter) = filter_from_value(log_level) { + self.filter = CustomEnvFilter(filter); + } self } /// Set the filter on the builder #[cfg(feature = "has-storage")] - pub fn with_filter(mut self, filter: EnvFilter) -> Self { - self.filter = Some(CustomEnvFilter(filter)); + pub fn with_filter(mut self, filter: CustomEnvFilter) -> Self { + self.filter = filter; self } /// Build a tracing dispatcher with the fmt subscriber (logs) and the chosen tracer subscriber pub fn build(self) -> Box { let registry = tracing_subscriber::registry(); - let registry = registry.with(self.filter.map(|filter| { - tracing_subscriber::fmt::layer() - .compact() - .with_ansi(true) - .with_span_events(FmtSpan::NONE) - .with_writer(std::io::stderr) - .with_filter(filter.0) - .boxed() - })); - let registry = registry.with(self.log_level.map(logs::new)); - let registry = registry.with(traces::new()); + + // Setup logging layer + let registry = registry.with(logs::new(self.filter.clone())); + + // Setup tracing layer + let registry = registry.with(traces::new(self.filter)); + Box::new(registry) } - /// tracing pipeline + /// Install the tracing dispatcher globally pub fn init(self) { self.build().init() } } +/// Create an EnvFilter from the given value. If the value is not a valid log level, it will be treated as EnvFilter directives. +pub fn filter_from_value(v: &str) -> Result { + match v { + // Don't show any logs at all + "none" => Ok(EnvFilter::default()), + // Check if we should show all log levels + "full" => Ok(EnvFilter::default().add_directive(Level::TRACE.into())), + // Otherwise, let's only show errors + "error" => Ok(EnvFilter::default().add_directive(Level::ERROR.into())), + // Specify the log level for each code area + "warn" | "info" | "debug" | "trace" => EnvFilter::builder() + .parse(format!("error,surreal={v},surrealdb={v},surrealdb::kvs::tx=error")), + // Let's try to parse the custom log level + _ => EnvFilter::builder().parse(v), + } +} + #[cfg(test)] mod tests { use opentelemetry::global::shutdown_tracer_provider; @@ -107,7 +128,7 @@ mod tests { ("OTEL_EXPORTER_OTLP_ENDPOINT", Some(otlp_endpoint.as_str())), ], || { - let _enter = telemetry::builder().build().set_default(); + let _enter = telemetry::builder().with_log_level("info").build().set_default(); println!("Sending span..."); @@ -123,7 +144,11 @@ mod tests { } println!("Waiting for request..."); - let req = req_rx.recv().await.expect("missing export request"); + let req = tokio::select! { + req = req_rx.recv() => req.expect("missing export request"), + _ = tokio::time::sleep(std::time::Duration::from_secs(1)) => panic!("timeout waiting for request"), + }; + let first_span = req.resource_spans.first().unwrap().scope_spans.first().unwrap().spans.first().unwrap(); assert_eq!("test-surreal-span", first_span.name); @@ -141,11 +166,10 @@ mod tests { temp_env::with_vars( vec![ ("SURREAL_TRACING_TRACER", Some("otlp")), - ("SURREAL_TRACING_FILTER", Some("debug")), ("OTEL_EXPORTER_OTLP_ENDPOINT", Some(otlp_endpoint.as_str())), ], || { - let _enter = telemetry::builder().build().set_default(); + let _enter = telemetry::builder().with_log_level("debug").build().set_default(); println!("Sending spans..."); @@ -169,7 +193,10 @@ mod tests { } println!("Waiting for request..."); - let req = req_rx.recv().await.expect("missing export request"); + let req = tokio::select! { + req = req_rx.recv() => req.expect("missing export request"), + _ = tokio::time::sleep(std::time::Duration::from_secs(1)) => panic!("timeout waiting for request"), + }; let spans = &req.resource_spans.first().unwrap().scope_spans.first().unwrap().spans; assert_eq!(1, spans.len()); diff --git a/src/telemetry/traces/mod.rs b/src/telemetry/traces/mod.rs index b562e435..841defbb 100644 --- a/src/telemetry/traces/mod.rs +++ b/src/telemetry/traces/mod.rs @@ -1,12 +1,15 @@ use tracing::Subscriber; use tracing_subscriber::Layer; +use crate::cli::validator::parser::env_filter::CustomEnvFilter; + pub mod otlp; +pub mod rpc; const TRACING_TRACER_VAR: &str = "SURREAL_TRACING_TRACER"; // Returns a tracer based on the value of the TRACING_TRACER_VAR env var -pub fn new() -> Option + Send + Sync>> +pub fn new(filter: CustomEnvFilter) -> Option + Send + Sync>> where S: Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a> + Send + Sync, { @@ -20,7 +23,7 @@ where // Init the registry with the OTLP tracer "otlp" => { debug!("Setup the OTLP tracer"); - Some(otlp::new()) + Some(otlp::new(filter)) } tracer => { panic!("unsupported tracer {}", tracer); diff --git a/src/telemetry/traces/otlp.rs b/src/telemetry/traces/otlp.rs index f820583c..8aa25050 100644 --- a/src/telemetry/traces/otlp.rs +++ b/src/telemetry/traces/otlp.rs @@ -1,18 +1,18 @@ use opentelemetry::sdk::trace::Tracer; use opentelemetry::trace::TraceError; use opentelemetry_otlp::WithExportConfig; -use tracing::{Level, Subscriber}; -use tracing_subscriber::{EnvFilter, Layer}; +use tracing::Subscriber; +use tracing_subscriber::Layer; -use crate::telemetry::OTEL_DEFAULT_RESOURCE; +use crate::{ + cli::validator::parser::env_filter::CustomEnvFilter, telemetry::OTEL_DEFAULT_RESOURCE, +}; -const TRACING_FILTER_VAR: &str = "SURREAL_TRACING_FILTER"; - -pub fn new() -> Box + Send + Sync> +pub fn new(filter: CustomEnvFilter) -> Box + Send + Sync> where S: Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a> + Send + Sync, { - tracing_opentelemetry::layer().with_tracer(tracer().unwrap()).with_filter(filter()).boxed() + tracing_opentelemetry::layer().with_tracer(tracer().unwrap()).with_filter(filter.0).boxed() } fn tracer() -> Result { @@ -24,16 +24,3 @@ fn tracer() -> Result { ) .install_batch(opentelemetry::runtime::Tokio) } - -/// Create a filter for the OTLP subscriber -/// -/// It creates an EnvFilter based on the TRACING_FILTER_VAR's value -/// -/// TRACING_FILTER_VAR accepts the same syntax as RUST_LOG -fn filter() -> EnvFilter { - EnvFilter::builder() - .with_env_var(TRACING_FILTER_VAR) - .with_default_directive(Level::INFO.into()) - .from_env() - .unwrap() -} diff --git a/src/telemetry/traces/rpc.rs b/src/telemetry/traces/rpc.rs new file mode 100644 index 00000000..6705a5b9 --- /dev/null +++ b/src/telemetry/traces/rpc.rs @@ -0,0 +1,31 @@ +use tracing::{field, Span}; +use uuid::Uuid; + +pub fn span_for_request(ws_id: &Uuid) -> Span { + let span = tracing::info_span!( + // Dynamic span names need to be 'recorded', can't be used on the macro. Use a static name here and overwrite later on + "rpc/call", + otel.name = field::Empty, + otel.kind = "server", + + // To be populated by the request handler when the method is known + rpc.method = field::Empty, + rpc.service = "surrealdb", + rpc.system = "jsonrpc", + + // JSON-RPC fields + rpc.jsonrpc.version = "2.0", + rpc.jsonrpc.request_id = field::Empty, + rpc.jsonrpc.error_code = field::Empty, + rpc.jsonrpc.error_message = field::Empty, + + // SurrealDB custom fields + ws.id = %ws_id, + + // Fields for error reporting + otel.status_code = field::Empty, + otel.status_message = field::Empty, + ); + + span +} diff --git a/tests/cli_integration.rs b/tests/cli_integration.rs index 47f7dabc..bdc35efc 100644 --- a/tests/cli_integration.rs +++ b/tests/cli_integration.rs @@ -5,6 +5,8 @@ mod common; use assert_fs::prelude::{FileTouch, FileWriteStr, PathChild}; use serial_test::serial; use std::fs; +use test_log::test; +use tracing::info; use common::{PASS, USER}; @@ -32,13 +34,14 @@ fn nonexistent_option() { assert!(common::run("version --turbo").output().is_err()); } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn all_commands() { // Commands without credentials when auth is disabled, should succeed let (addr, _server) = common::start_server(false, false, true).await.unwrap(); let creds = ""; // Anonymous user - // Create a record + + info!("* Create a record"); { let args = format!("sql --conn http://{addr} {creds} --ns N --db D --multi"); assert_eq!( @@ -48,7 +51,7 @@ async fn all_commands() { ); } - // Export to stdout + info!("* Export to stdout"); { let args = format!("export --conn http://{addr} {creds} --ns N --db D -"); let output = common::run(&args).output().expect("failed to run stdout export: {args}"); @@ -56,7 +59,7 @@ async fn all_commands() { assert!(output.contains("UPDATE thing:one CONTENT { id: thing:one };")); } - // Export to file + info!("* Export to file"); let exported = { let exported = common::tmp_file("exported.surql"); let args = format!("export --conn http://{addr} {creds} --ns N --db D {exported}"); @@ -64,13 +67,13 @@ async fn all_commands() { exported }; - // Import the exported file + info!("* Import the exported file"); { let args = format!("import --conn http://{addr} {creds} --ns N --db D2 {exported}"); common::run(&args).output().expect("failed to run import: {args}"); } - // Query from the import (pretty-printed this time) + info!("* Query from the import (pretty-printed this time)"); { let args = format!("sql --conn http://{addr} {creds} --ns N --db D2 --pretty"); assert_eq!( @@ -80,7 +83,7 @@ async fn all_commands() { ); } - // Unfinished backup CLI + info!("* Unfinished backup CLI"); { let file = common::tmp_file("backup.db"); let args = format!("backup {creds} http://{addr} {file}"); @@ -90,7 +93,7 @@ async fn all_commands() { assert_eq!(fs::read_to_string(file).unwrap(), "Save"); } - // Multi-statement (and multi-line) query including error(s) over WS + info!("* Multi-statement (and multi-line) query including error(s) over WS"); { let args = format!("sql --conn ws://{addr} {creds} --ns N3 --db D3 --multi --pretty"); let output = common::run(&args) @@ -113,7 +116,7 @@ async fn all_commands() { assert!(output.contains("thing:also_success"), "missing also_success in {output}") } - // Multi-statement (and multi-line) transaction including error(s) over WS + info!("* Multi-statement (and multi-line) transaction including error(s) over WS"); { let args = format!("sql --conn ws://{addr} {creds} --ns N4 --db D4 --multi --pretty"); let output = common::run(&args) @@ -137,7 +140,7 @@ async fn all_commands() { assert!(output.contains("rgument"), "missing argument error in {output}"); } - // Pass neither ns nor db + info!("* Pass neither ns nor db"); { let args = format!("sql --conn http://{addr} {creds}"); let output = common::run(&args) @@ -147,7 +150,7 @@ async fn all_commands() { assert!(output.contains("thing:one"), "missing thing:one in {output}"); } - // Pass only ns + info!("* Pass only ns"); { let args = format!("sql --conn http://{addr} {creds} --ns N5"); let output = common::run(&args) @@ -157,26 +160,36 @@ async fn all_commands() { assert!(output.contains("thing:one"), "missing thing:one in {output}"); } - // Pass only db and expect an error + info!("* Pass only db and expect an error"); { let args = format!("sql --conn http://{addr} {creds} --db D5"); common::run(&args).output().expect_err("only db"); } } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn start_tls() { - let (_, server) = common::start_server(false, true, false).await.unwrap(); + // Capute the server's stdout/stderr + temp_env::async_with_vars( + [ + ("SURREAL_TEST_SERVER_STDOUT", Some("piped")), + ("SURREAL_TEST_SERVER_STDERR", Some("piped")), + ], + async { + let (_, server) = common::start_server(false, true, false).await.unwrap(); - std::thread::sleep(std::time::Duration::from_millis(2000)); - let output = server.kill().output().err().unwrap(); + std::thread::sleep(std::time::Duration::from_millis(2000)); + let output = server.kill().output().err().unwrap(); - // Test the crt/key args but the keys are self signed so don't actually connect. - assert!(output.contains("Started web server"), "couldn't start web server: {output}"); + // Test the crt/key args but the keys are self signed so don't actually connect. + assert!(output.contains("Started web server"), "couldn't start web server: {output}"); + }, + ) + .await; } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn with_root_auth() { // Commands with credentials when auth is enabled, should succeed @@ -184,7 +197,7 @@ async fn with_root_auth() { let creds = format!("--user {USER} --pass {PASS}"); let sql_args = format!("sql --conn http://{addr} --multi --pretty"); - // Can query /sql over HTTP + info!("* Query over HTTP"); { let args = format!("{sql_args} {creds}"); let input = "INFO FOR ROOT;"; @@ -192,7 +205,7 @@ async fn with_root_auth() { assert!(output.is_ok(), "failed to query over HTTP: {}", output.err().unwrap()); } - // Can query /sql over WS + info!("* Query over WS"); { let args = format!("sql --conn ws://{addr} --multi --pretty {creds}"); let input = "INFO FOR ROOT;"; @@ -200,7 +213,7 @@ async fn with_root_auth() { assert!(output.is_ok(), "failed to query over WS: {}", output.err().unwrap()); } - // KV user can do exports + info!("* Root user can do exports"); let exported = { let exported = common::tmp_file("exported.surql"); let args = format!("export --conn http://{addr} {creds} --ns N --db D {exported}"); @@ -209,13 +222,13 @@ async fn with_root_auth() { exported }; - // KV user can do imports + info!("* Root user can do imports"); { let args = format!("import --conn http://{addr} {creds} --ns N --db D2 {exported}"); common::run(&args).output().unwrap_or_else(|_| panic!("failed to run import: {args}")); } - // KV user can do backups + info!("* Root user can do backups"); { let file = common::tmp_file("backup.db"); let args = format!("backup {creds} http://{addr} {file}"); @@ -226,7 +239,7 @@ async fn with_root_auth() { } } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn with_anon_auth() { // Commands without credentials when auth is enabled, should fail @@ -234,7 +247,7 @@ async fn with_anon_auth() { let creds = ""; // Anonymous user let sql_args = format!("sql --conn http://{addr} --multi --pretty"); - // Can query /sql over HTTP + info!("* Query over HTTP"); { let args = format!("{sql_args} {creds}"); let input = ""; @@ -242,7 +255,7 @@ async fn with_anon_auth() { assert!(output.is_ok(), "anonymous user should be able to query: {:?}", output); } - // Can query /sql over HTTP + info!("* Query over WS"); { let args = format!("sql --conn ws://{addr} --multi --pretty {creds}"); let input = ""; @@ -250,7 +263,7 @@ async fn with_anon_auth() { assert!(output.is_ok(), "anonymous user should be able to query: {:?}", output); } - // Can't do exports + info!("* Can't do exports"); { let args = format!("export --conn http://{addr} {creds} --ns N --db D -"); let output = common::run(&args).output(); @@ -261,7 +274,7 @@ async fn with_anon_auth() { ); } - // Can't do imports + info!("* Can't do imports"); { let tmp_file = common::tmp_file("exported.surql"); let args = format!("import --conn http://{addr} {creds} --ns N --db D2 {tmp_file}"); @@ -273,7 +286,7 @@ async fn with_anon_auth() { ); } - // Can't do backups + info!("* Can't do backups"); { let args = format!("backup {creds} http://{addr}"); let output = common::run(&args).output(); diff --git a/tests/common/mod.rs b/tests/common/mod.rs index bd190b44..ca24026a 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,10 +1,18 @@ #![allow(dead_code)] +use futures_util::{SinkExt, StreamExt, TryStreamExt}; use rand::{thread_rng, Rng}; +use serde::{Deserialize, Serialize}; +use serde_json::json; use std::error::Error; -use std::fs; +use std::fs::File; use std::path::Path; use std::process::{Command, Stdio}; +use std::{env, fs}; +use tokio::net::TcpStream; use tokio::time; +use tokio_tungstenite::tungstenite::Message; +use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; +use tracing::{error, info}; pub const USER: &str = "root"; pub const PASS: &str = "root"; @@ -52,7 +60,12 @@ impl Drop for Child { } } -pub fn run_internal>(args: &str, current_dir: Option

) -> Child { +pub fn run_internal>( + args: &str, + current_dir: Option

, + stdout: Stdio, + stderr: Stdio, +) -> Child { let mut path = std::env::current_exe().unwrap(); assert!(path.pop()); if path.ends_with("deps") { @@ -68,8 +81,8 @@ pub fn run_internal>(args: &str, current_dir: Option

) -> Child } cmd.env_clear(); cmd.stdin(Stdio::piped()); - cmd.stdout(Stdio::piped()); - cmd.stderr(Stdio::piped()); + cmd.stdout(stdout); + cmd.stderr(stderr); cmd.args(args.split_ascii_whitespace()); Child { inner: Some(cmd.spawn().unwrap()), @@ -78,12 +91,12 @@ pub fn run_internal>(args: &str, current_dir: Option

) -> Child /// Run the CLI with the given args pub fn run(args: &str) -> Child { - run_internal::(args, None) + run_internal::(args, None, Stdio::piped(), Stdio::piped()) } /// Run the CLI with the given args inside a temporary directory pub fn run_in_dir>(args: &str, current_dir: P) -> Child { - run_internal(args, Some(current_dir)) + run_internal(args, Some(current_dir), Stdio::piped(), Stdio::piped()) } pub fn tmp_file(name: &str) -> String { @@ -91,6 +104,19 @@ pub fn tmp_file(name: &str) -> String { path.to_string_lossy().into_owned() } +fn parse_server_stdio_from_var(var: &str) -> Result> { + match env::var(var).as_deref() { + Ok("inherit") => Ok(Stdio::inherit()), + Ok("null") => Ok(Stdio::null()), + Ok("piped") => Ok(Stdio::piped()), + Ok(val) if val.starts_with("file://") => { + Ok(Stdio::from(File::create(val.trim_start_matches("file://"))?)) + } + Ok(val) => Err(format!("Unsupported stdio value: {val:?}").into()), + _ => Ok(Stdio::null()), + } +} + pub async fn start_server( auth: bool, tls: bool, @@ -118,11 +144,14 @@ pub async fn start_server( extra_args.push_str(" --auth"); } - let start_args = format!("start --bind {addr} memory --no-banner --log info --user {USER} --pass {PASS} {extra_args}"); + let start_args = format!("start --bind {addr} memory --no-banner --log trace --user {USER} --pass {PASS} {extra_args}"); - println!("starting server with args: {start_args}"); + info!("starting server with args: {start_args}"); - let server = run(&start_args); + // Configure where the logs go when running the test + let stdout = parse_server_stdio_from_var("SURREAL_TEST_SERVER_STDOUT")?; + let stderr = parse_server_stdio_from_var("SURREAL_TEST_SERVER_STDERR")?; + let server = run_internal::(&start_args, None, stdout, stderr); if !wait_is_ready { return Ok((addr, server)); @@ -130,17 +159,178 @@ pub async fn start_server( // Wait 5 seconds for the server to start let mut interval = time::interval(time::Duration::from_millis(500)); - println!("Waiting for server to start..."); + info!("Waiting for server to start..."); for _i in 0..10 { interval.tick().await; if run(&format!("isready --conn http://{addr}")).output().is_ok() { - println!("Server ready!"); + info!("Server ready!"); return Ok((addr, server)); } } let server_out = server.kill().output().err().unwrap(); - println!("server output: {server_out}"); + error!("server output: {server_out}"); Err("server failed to start".into()) } + +type WsStream = WebSocketStream>; + +pub async fn connect_ws(addr: &str) -> Result> { + let url = format!("ws://{}/rpc", addr); + let (ws_stream, _) = connect_async(url).await?; + Ok(ws_stream) +} + +pub async fn ws_send_msg( + socket: &mut WsStream, + msg_req: String, +) -> Result> { + // Use JSON format by default + ws_send_msg_with_fmt(socket, msg_req, Format::Json).await +} + +pub enum Format { + Json, + Cbor, + Pack, +} + +pub async fn ws_send_msg_with_fmt( + socket: &mut WsStream, + msg_req: String, + response_format: Format, +) -> Result> { + tokio::select! { + _ = time::sleep(time::Duration::from_millis(500)) => { + return Err("timeout waiting for the request to be sent".into()); + } + res = socket.send(Message::Text(msg_req)) => { + if let Err(err) = res { + return Err(format!("Error sending the message: {}", err).into()); + } + } + } + + let mut f = socket.try_filter(|msg| match response_format { + Format::Json => futures_util::future::ready(msg.is_text()), + Format::Pack | Format::Cbor => futures_util::future::ready(msg.is_binary()), + }); + + tokio::select! { + _ = time::sleep(time::Duration::from_millis(2000)) => { + Err("timeout waiting for the response".into()) + } + res = f.select_next_some() => { + match response_format { + Format::Json => Ok(serde_json::from_str(&res?.to_string())?), + Format::Cbor => Ok(serde_cbor::from_slice(&res?.into_data())?), + Format::Pack => Ok(serde_pack::from_slice(&res?.into_data())?), + } + } + } +} + +#[derive(Serialize, Deserialize)] +struct SigninParams<'a> { + user: &'a str, + pass: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + ns: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + db: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + sc: Option<&'a str>, +} +#[derive(Serialize, Deserialize)] +struct UseParams<'a> { + #[serde(skip_serializing_if = "Option::is_none")] + ns: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + db: Option<&'a str>, +} + +pub async fn ws_signin( + socket: &mut WsStream, + user: &str, + pass: &str, + ns: Option<&str>, + db: Option<&str>, + sc: Option<&str>, +) -> Result> { + let json = json!({ + "id": "1", + "method": "signin", + "params": [ + SigninParams { user, pass, ns, db, sc } + ], + }); + + let msg = ws_send_msg(socket, serde_json::to_string(&json).unwrap()).await?; + match msg.as_object() { + Some(obj) if obj.keys().all(|k| ["id", "error"].contains(&k.as_str())) => { + Err(format!("unexpected error from query request: {:?}", obj.get("error")).into()) + } + Some(obj) if obj.keys().all(|k| ["id", "result"].contains(&k.as_str())) => { + Ok(obj.get("result").unwrap().as_str().unwrap_or_default().to_owned()) + } + _ => { + error!("{:?}", msg.as_object().unwrap().keys().collect::>()); + Err(format!("unexpected response: {:?}", msg).into()) + } + } +} + +pub async fn ws_query( + socket: &mut WsStream, + query: &str, +) -> Result, Box> { + let json = json!({ + "id": "1", + "method": "query", + "params": [query], + }); + + let msg = ws_send_msg(socket, serde_json::to_string(&json).unwrap()).await?; + + match msg.as_object() { + Some(obj) if obj.keys().all(|k| ["id", "error"].contains(&k.as_str())) => { + Err(format!("unexpected error from query request: {:?}", obj.get("error")).into()) + } + Some(obj) if obj.keys().all(|k| ["id", "result"].contains(&k.as_str())) => { + Ok(obj.get("result").unwrap().as_array().unwrap().to_owned()) + } + _ => { + error!("{:?}", msg.as_object().unwrap().keys().collect::>()); + Err(format!("unexpected response: {:?}", msg).into()) + } + } +} + +pub async fn ws_use( + socket: &mut WsStream, + ns: Option<&str>, + db: Option<&str>, +) -> Result> { + let json = json!({ + "id": "1", + "method": "use", + "params": [ + ns, db + ], + }); + + let msg = ws_send_msg(socket, serde_json::to_string(&json).unwrap()).await?; + match msg.as_object() { + Some(obj) if obj.keys().all(|k| ["id", "error"].contains(&k.as_str())) => { + Err(format!("unexpected error from query request: {:?}", obj.get("error")).into()) + } + Some(obj) if obj.keys().all(|k| ["id", "result"].contains(&k.as_str())) => { + Ok(obj.get("result").unwrap().to_owned()) + } + _ => { + error!("{:?}", msg.as_object().unwrap().keys().collect::>()); + Err(format!("unexpected response: {:?}", msg).into()) + } + } +} diff --git a/tests/http_integration.rs b/tests/http_integration.rs index c2016a95..4d6f2662 100644 --- a/tests/http_integration.rs +++ b/tests/http_integration.rs @@ -7,10 +7,11 @@ use http::{header, Method}; use reqwest::Client; use serde_json::json; use serial_test::serial; +use test_log::test; use crate::common::{PASS, USER}; -#[tokio::test] +#[test(tokio::test)] #[serial] async fn basic_auth() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -52,7 +53,7 @@ async fn basic_auth() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn bearer_auth() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -133,14 +134,14 @@ async fn bearer_auth() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn client_ip_extractor() -> Result<(), Box> { // TODO: test the client IP extractor Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn export_endpoint() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -184,7 +185,7 @@ async fn export_endpoint() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn health_endpoint() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -196,7 +197,7 @@ async fn health_endpoint() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn import_endpoint() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -269,7 +270,7 @@ async fn import_endpoint() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn rpc_endpoint() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -303,7 +304,7 @@ async fn rpc_endpoint() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn signin_endpoint() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -372,7 +373,7 @@ async fn signin_endpoint() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn signup_endpoint() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -425,13 +426,17 @@ async fn signup_endpoint() -> Result<(), Box> { assert_eq!(res.status(), 200, "body: {}", res.text().await?); let body: serde_json::Value = serde_json::from_str(&res.text().await?).unwrap(); - assert!(!body["token"].as_str().unwrap().to_string().is_empty(), "body: {}", body); + assert!( + body["token"].as_str().unwrap().starts_with("eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9"), + "body: {}", + body + ); } Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn sql_endpoint() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -543,7 +548,7 @@ async fn sql_endpoint() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn sync_endpoint() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -577,7 +582,7 @@ async fn sync_endpoint() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn version_endpoint() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -619,7 +624,7 @@ async fn seed_table( Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn key_endpoint_select_all() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -696,7 +701,7 @@ async fn key_endpoint_select_all() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn key_endpoint_create_all() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -759,7 +764,7 @@ async fn key_endpoint_create_all() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn key_endpoint_update_all() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -829,7 +834,7 @@ async fn key_endpoint_update_all() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn key_endpoint_modify_all() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -899,7 +904,7 @@ async fn key_endpoint_modify_all() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn key_endpoint_delete_all() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -953,7 +958,7 @@ async fn key_endpoint_delete_all() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn key_endpoint_select_one() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -994,7 +999,7 @@ async fn key_endpoint_select_one() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn key_endpoint_create_one() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -1091,7 +1096,7 @@ async fn key_endpoint_create_one() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn key_endpoint_update_one() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -1164,7 +1169,7 @@ async fn key_endpoint_update_one() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn key_endpoint_modify_one() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); @@ -1242,7 +1247,7 @@ async fn key_endpoint_modify_one() -> Result<(), Box> { Ok(()) } -#[tokio::test] +#[test(tokio::test)] #[serial] async fn key_endpoint_delete_one() -> Result<(), Box> { let (addr, _server) = common::start_server(true, false, true).await.unwrap(); diff --git a/tests/ws_integration.rs b/tests/ws_integration.rs new file mode 100644 index 00000000..e5fe9bd8 --- /dev/null +++ b/tests/ws_integration.rs @@ -0,0 +1,1109 @@ +// cargo test --package surreal --bin surreal --no-default-features --features storage-mem --test ws_integration -- --nocapture + +mod common; + +use serde_json::json; +use serial_test::serial; +use test_log::test; + +use crate::common::{PASS, USER}; + +#[test(tokio::test)] +#[serial] +async fn ping() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // Send command + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "ping", + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res.is_object(), "result: {:?}", res); + let res = res.as_object().unwrap(); + assert!(res.keys().all(|k| ["id", "result"].contains(&k.as_str())), "result: {:?}", res); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn info() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Setup operations + // + let res = common::ws_query(socket, "DEFINE TABLE user PERMISSIONS FULL").await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_query( + socket, + r#" + DEFINE SCOPE scope SESSION 24h + SIGNUP ( CREATE user SET user = $user, pass = crypto::argon2::generate($pass) ) + SIGNIN ( SELECT * FROM user WHERE user = $user AND crypto::argon2::compare(pass, $pass) ) + ; + "#, + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_query( + socket, + r#" + CREATE user CONTENT { + user: 'user', + pass: crypto::argon2::generate('pass') + }; + "#, + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // Sign in + let res = common::ws_signin(socket, "user", "pass", Some("N"), Some("D"), Some("scope")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // Send the info command + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "info", + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // Verify the response contains the expected info + let res = res.unwrap(); + assert!(res["result"].is_object(), "result: {:?}", res); + let res = res["result"].as_object().unwrap(); + assert_eq!(res["user"], "user", "result: {:?}", res); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn signup() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // Setup scope + let res = common::ws_query(socket, r#" + DEFINE SCOPE scope SESSION 24h + SIGNUP ( CREATE user SET email = $email, pass = crypto::argon2::generate($pass) ) + SIGNIN ( SELECT * FROM user WHERE email = $email AND crypto::argon2::compare(pass, $pass) ) + ;"#).await; + assert!(res.is_ok(), "result: {:?}", res); + + // Signup + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "signup", + "params": [{ + "ns": "N", + "db": "D", + "sc": "scope", + "email": "email@email.com", + "pass": "pass", + }], + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res.is_object(), "result: {:?}", res); + let res = res.as_object().unwrap(); + + // Verify response contains no error + assert!(res.keys().all(|k| ["id", "result"].contains(&k.as_str())), "result: {:?}", res); + + // Verify it returns a token + assert!(res["result"].is_string(), "result: {:?}", res); + + let res = res["result"].as_str().unwrap(); + assert!(res.starts_with("eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9"), "result: {}", res); + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn signin() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // Setup scope + let res = common::ws_query(socket, r#" + DEFINE SCOPE scope SESSION 24h + SIGNUP ( CREATE user SET email = $email, pass = crypto::argon2::generate($pass) ) + SIGNIN ( SELECT * FROM user WHERE email = $email AND crypto::argon2::compare(pass, $pass) ) + ;"#).await; + assert!(res.is_ok(), "result: {:?}", res); + + // Signup + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "signup", + "params": [{ + "ns": "N", + "db": "D", + "sc": "scope", + "email": "email@email.com", + "pass": "pass", + }], + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // Sign in + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "signin", + "params": [{ + "ns": "N", + "db": "D", + "sc": "scope", + "email": "email@email.com", + "pass": "pass", + }], + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res.is_object(), "result: {:?}", res); + let res = res.as_object().unwrap(); + + // Verify response contains no error + assert!(res.keys().all(|k| ["id", "result"].contains(&k.as_str())), "result: {:?}", res); + + // Verify it returns a token + assert!(res["result"].is_string(), "result: {:?}", res); + let res = res["result"].as_str().unwrap(); + assert!(res.starts_with("eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9"), "result: {}", res); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn invalidate() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + + // Verify we have a ROOT session + let res = common::ws_query(socket, "DEFINE NAMESPACE NS").await; + assert!(res.is_ok(), "result: {:?}", res); + + // Invalidate session + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "invalidate", + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // Verify we invalidated the root session + let res = common::ws_query(socket, "DEFINE NAMESPACE NS2").await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + + assert_eq!(res[0]["status"], "ERR", "result: {:?}", res); + assert_eq!( + res[0]["result"], "IAM error: Not enough permissions to perform this action", + "result: {:?}", + res + ); + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn authenticate() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let token = res.unwrap(); + + // Reconnect so we start with an empty session + socket.close(None).await?; + let socket = &mut common::connect_ws(&addr).await?; + + // + // Authenticate with the token + // + + // Send command + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "authenticate", + "params": [ + token, + + ], + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // Verify we have a ROOT session + let res = common::ws_query(socket, "DEFINE NAMESPACE D2").await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert_eq!(res[0]["status"], "OK", "result: {:?}", res); + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn kill() -> Result<(), Box> { + // TODO: implement + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn live() -> Result<(), Box> { + // TODO: implement + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn let_and_set() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // Define variable using let + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "let", + "params": [ + "let_var", "let_value", + ], + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // Define variable using set + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "set", + "params": [ + "set_var", "set_value", + ], + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // Verify the variables are set + let res = common::ws_query(socket, "SELECT * FROM $let_var, $set_var").await?; + assert_eq!( + res[0]["result"], + serde_json::to_value(["let_value", "set_value"]).unwrap(), + "result: {:?}", + res + ); + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn unset() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // Define variable + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "let", + "params": [ + "let_var", "let_value", + ], + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // Verify the variable is set + let res = common::ws_query(socket, "SELECT * FROM $let_var").await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = res[0]["result"].as_array().unwrap(); + + assert_eq!(res[0], "let_value", "result: {:?}", res); + + // Unset variable + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "unset", + "params": [ + "let_var", + ], + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // Verify the variable is unset + let res = common::ws_query(socket, "SELECT * FROM $let_var").await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = res[0]["result"].as_array().unwrap(); + + assert!(res[0].is_null(), "result: {:?}", res); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn select() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Setup the database + // + let res = common::ws_query(socket, "CREATE foo").await; + assert!(res.is_ok(), "result: {:?}", res); + + // Select data + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "select", + "params": [ + "foo", + ], + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res["result"].is_array(), "result: {:?}", res); + let res = res["result"].as_array().unwrap(); + + // Verify the response contains the output of the select + assert_eq!(res.len(), 1, "result: {:?}", res); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn insert() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // Insert data + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "insert", + "params": [ + "table", + { + "name": "foo", + "value": "bar", + } + ], + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // Verify the data was inserted and can be queried + let res = common::ws_query(socket, "SELECT * FROM table").await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = res[0]["result"].as_array().unwrap(); + + assert_eq!(res[0]["name"], "foo", "result: {:?}", res); + assert_eq!(res[0]["value"], "bar", "result: {:?}", res); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn create() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // Insert data + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "create", + "params": [ + "table", + ], + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // Verify the data was created and can be queried + let res = common::ws_query(socket, "SELECT * FROM table").await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = res[0]["result"].as_array().unwrap(); + assert_eq!(res.len(), 1, "result: {:?}", res); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn update() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Setup the database + // + let res = common::ws_query(socket, r#"CREATE table SET name = "foo""#).await; + assert!(res.is_ok(), "result: {:?}", res); + + // Insert data + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "update", + "params": [ + "table", + { + "value": "bar", + } + ], + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // Verify the data was updated + let res = common::ws_query(socket, "SELECT * FROM table").await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = &res[0]["result"].as_array().unwrap()[0]; + assert!(res["name"].is_null(), "result: {:?}", res); + assert_eq!(res["value"], "bar", "result: {:?}", res); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn change_and_merge() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Setup the database + // + let res = common::ws_query(socket, "CREATE foo").await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = &res[0]["result"].as_array().unwrap()[0]; + + assert!(res["id"].is_string(), "result: {:?}", res); + let what = res["id"].as_str().unwrap(); + + // + // Change / Marge data + // + + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "change", + "params": [ + what, { + "name_for_change": "foo", + "value_for_change": "bar", + } + ] + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "merge", + "params": [ + what, { + "name_for_merge": "foo", + "value_for_merge": "bar", + } + ] + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Get the data and verify the output + // + let res = common::ws_query(socket, &format!("SELECT * FROM foo WHERE id = {what}")).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = &res[0]["result"].as_array().unwrap()[0]; + + assert_eq!(res["id"], what); + assert_eq!(res["name_for_change"], "foo"); + assert_eq!(res["value_for_change"], "bar"); + assert_eq!(res["name_for_merge"], "foo"); + assert_eq!(res["value_for_merge"], "bar"); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn modify_and_patch() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Setup the database + // + let res = + common::ws_query(socket, r#"CREATE table SET original_name = "oritinal_value""#).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = &res[0]["result"].as_array().unwrap()[0]; + + let what = res["id"].as_str().unwrap(); + + // + // Modify data + // + + let ops = json!([ + { + "op": "add", + "path": "modify_name", + "value": "modify_value" + }, + { + "op": "remove", + "path": "original_name", + } + ]); + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "modify", + "params": [ + what, ops + ] + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res["result"].is_array(), "result: {:?}", res); + let res = res["result"].as_array().unwrap(); + assert_eq!(res.len(), ops.as_array().unwrap().len(), "result: {:?}", res); + + // + // Patch data + // + + let ops = json!([ + { + "op": "add", + "path": "patch_name", + "value": "patch_value" + } + ]); + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "patch", + "params": [ + what, ops + ] + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res["result"].is_array(), "result: {:?}", res); + let res = res["result"].as_array().unwrap(); + assert_eq!(res.len(), ops.as_array().unwrap().len(), "result: {:?}", res); + + // + // Get the data and verify the output + // + let res = common::ws_query(socket, &format!("SELECT * FROM table WHERE id = {what}")).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = &res[0]["result"].as_array().unwrap()[0]; + + assert_eq!(res["id"], what); + assert!(res["original_name"].is_null()); + assert_eq!(res["modify_name"], "modify_value"); + assert_eq!(res["patch_name"], "patch_value"); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn delete() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Setup the database + // + let res = common::ws_query(socket, "CREATE table:id").await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Verify the data was created and can be queried + // + let res = common::ws_query(socket, "SELECT * FROM table:id").await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = res[0]["result"].as_array().unwrap(); + + assert_eq!(res.len(), 1, "result: {:?}", res); + + // + // Delete data + // + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "delete", + "params": [ + "table:id" + ] + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Verify the data was deleted + // + let res = common::ws_query(socket, "SELECT * FROM table:id").await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = res[0]["result"].as_array().unwrap(); + + assert_eq!(res.len(), 0, "result: {:?}", res); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn format_json() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Setup the database + // + let res = common::ws_query(socket, "CREATE table:id").await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Test JSON format + // + + // Change format + let msg = json!({ + "id": "1", + "method": "format", + "params": [ + "json" + ] + }); + let res = common::ws_send_msg(socket, serde_json::to_string(&msg).unwrap()).await; + assert!(res.is_ok(), "result: {:?}", res); + + // Query data + let res = common::ws_query(socket, "SELECT * FROM table:id").await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = res[0]["result"].as_array().unwrap(); + assert_eq!(res.len(), 1, "result: {:?}", res); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn format_cbor() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Setup the database + // + let res = common::ws_query(socket, "CREATE table:id").await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Test CBOR format + // + + // Change format + let msg = serde_json::to_string(&json!({ + "id": "1", + "method": "format", + "params": [ + "cbor" + ] + })) + .unwrap(); + let res = common::ws_send_msg(socket, msg).await; + assert!(res.is_ok(), "result: {:?}", res); + + // Query data + let msg = serde_json::to_string(&json!({ + "id": "1", + "method": "query", + "params": [ + "SELECT * FROM table:id" + ] + })) + .unwrap(); + + let res = common::ws_send_msg_with_fmt(socket, msg, common::Format::Cbor).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res["result"].is_array(), "result: {:?}", res); + let res = res["result"].as_array().unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = res[0]["result"].as_array().unwrap(); + assert_eq!(res.len(), 1, "result: {:?}", res); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn format_pack() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Setup the database + // + let res = common::ws_query(socket, "CREATE table:id").await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Test PACK format + // + + // Change format + let msg = serde_json::to_string(&json!({ + "id": "1", + "method": "format", + "params": [ + "pack" + ] + })) + .unwrap(); + let res = common::ws_send_msg(socket, msg).await; + assert!(res.is_ok(), "result: {:?}", res); + + // Query data + let msg = serde_json::to_string(&json!({ + "id": "1", + "method": "query", + "params": [ + "SELECT * FROM table:id" + ] + })) + .unwrap(); + + let res = common::ws_send_msg_with_fmt(socket, msg, common::Format::Pack).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res["result"].is_array(), "result: {:?}", res); + let res = res["result"].as_array().unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = res[0]["result"].as_array().unwrap(); + assert_eq!(res.len(), 1, "result: {:?}", res); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn query() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // + // Prepare the connection + // + let res = common::ws_signin(socket, USER, PASS, None, None, None).await; + assert!(res.is_ok(), "result: {:?}", res); + let res = common::ws_use(socket, Some("N"), Some("D")).await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Run a CREATE query + // + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "query", + "params": [ + "CREATE foo", + ] + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + + // + // Verify the data was created and can be queried + // + let res = common::ws_query(socket, "SELECT * FROM foo").await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res[0]["result"].is_array(), "result: {:?}", res); + let res = res[0]["result"].as_array().unwrap(); + assert_eq!(res.len(), 1, "result: {:?}", res); + + Ok(()) +} + +#[test(tokio::test)] +#[serial] +async fn version() -> Result<(), Box> { + let (addr, _server) = common::start_server(true, false, true).await.unwrap(); + let socket = &mut common::connect_ws(&addr).await?; + + // Send command + let res = common::ws_send_msg( + socket, + serde_json::to_string(&json!({ + "id": "1", + "method": "version", + "params": [], + })) + .unwrap(), + ) + .await; + assert!(res.is_ok(), "result: {:?}", res); + let res = res.unwrap(); + assert!(res["result"].is_string(), "result: {:?}", res); + let res = res["result"].as_str().unwrap(); + + // Verify response + assert!(res.starts_with("surrealdb-"), "result: {}", res); + Ok(()) +}