Use custom serialization format

Cargo.lock

@@ -710,6 +710,17 @@ version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
 
+[[package]]
+name = "bung"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da8bcf29331f126c3b4f20a6698909d58004290723aac75e3eafab41ae3c2953"
+dependencies = [
+ "byteorder",
+ "rmp",
+ "serde",
+]
+
 [[package]]
 name = "byteorder"
 version = "1.4.3"
@@ -3769,6 +3780,7 @@ version = "1.0.0-beta.8"
 dependencies = [
  "argon2",
  "base64 0.21.0",
+ "bung",
  "bytes",
  "chrono",
  "clap",
@@ -3815,6 +3827,7 @@ dependencies = [
  "async-recursion",
  "bcrypt",
  "bigdecimal",
+ "bung",
  "chrono",
  "dmp",
  "echodb",
@@ -3839,7 +3852,6 @@ dependencies = [
  "rand 0.8.5",
  "regex",
  "reqwest",
- "rmp-serde",
  "rocksdb",
  "rquickjs",
  "rustls",
@@ -3870,9 +3882,9 @@ dependencies = [
 
 [[package]]
 name = "surrealdb-derive"
-version = "0.5.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "552bb4f9eb49f493b14d79d989ea9ecb53ee8cee0aad529e289faca72fb78b14"
+checksum = "dfb39273220b1d4cd0936abb8d1e9ec0eb673e17044169cb4604345576b1e0ef"
 dependencies = [
  "quote",
  "syn 1.0.109",

Cargo.toml

@@ -27,6 +27,7 @@ codegen-units = 1
 [dependencies]
 argon2 = "0.5.0"
 base64 = "0.21.0"
+bung = "0.1.0"
 bytes = "1.4.0"
 chrono = { version = "0.4.24", features = ["serde"] }
 clap = { version = "3.2.23", features = ["env"] }

lib/Cargo.toml

@@ -56,9 +56,10 @@ ascii = { version = "0.3.2", package = "any_ascii" }
 async-recursion = "1.0.4"
 bcrypt = "0.14.0"
 bigdecimal = { version = "0.3.0", features = ["serde", "string-only"] }
+bung = "0.1.0"
 channel = { version = "1.8.0", package = "async-channel" }
 chrono = { version = "0.4.24", features = ["serde"] }
-derive = { version = "0.5.0", package = "surrealdb-derive" }
+derive = { version = "0.6.0", package = "surrealdb-derive" }
 dmp = "0.1.3"
 echodb = { version = "0.4.0", optional = true }
 executor = { version = "1.5.0", package = "async-executor" }
@@ -74,7 +75,6 @@ js = { version = "0.1.7", package = "rquickjs", features = ["array-buffer", "bin
 lexicmp = "0.1.0"
 log = "0.4.17"
 md-5 = "0.10.5"
-msgpack = { version = "1.1.1", package = "rmp-serde" }
 nanoid = "0.4.0"
 native-tls = { version = "0.2.11", optional = true }
 nom = { version = "7.1.3", features = ["alloc"] }

lib/src/api/engine/remote/http/mod.rs

@@ -104,7 +104,7 @@ impl Surreal<Client> {
 
 pub(crate) fn default_headers() -> HeaderMap {
 	let mut headers = HeaderMap::new();
-	headers.insert(ACCEPT, HeaderValue::from_static("application/cork"));
+	headers.insert(ACCEPT, HeaderValue::from_static("application/bung"));
 	headers
 }
 
@@ -168,7 +168,7 @@ async fn submit_auth(request: RequestBuilder) -> Result<Value> {
 	let response = request.send().await?.error_for_status()?;
 	let bytes = response.bytes().await?;
 	let response: AuthResponse =
-		msgpack::from_slice(&bytes).map_err(|error| Error::ResponseFromBinary {
+		bung::from_slice(&bytes).map_err(|error| Error::ResponseFromBinary {
 			binary: bytes.to_vec(),
 			error,
 		})?;
@@ -180,7 +180,7 @@ async fn query(request: RequestBuilder) -> Result<QueryResponse> {
 	let response = request.send().await?.error_for_status()?;
 	let bytes = response.bytes().await?;
 	let responses: Vec<HttpQueryResponse> =
-		msgpack::from_slice(&bytes).map_err(|error| Error::ResponseFromBinary {
+		bung::from_slice(&bytes).map_err(|error| Error::ResponseFromBinary {
 			binary: bytes.to_vec(),
 			error,
 		})?;

lib/src/api/engine/remote/ws/native.rs

@@ -422,7 +422,7 @@ impl Response {
 				trace!(target: LOG, "Received an unexpected text message; {text}");
 				Ok(None)
 			}
-			Message::Binary(binary) => msgpack::from_slice(&binary).map(Some).map_err(|error| {
+			Message::Binary(binary) => bung::from_slice(&binary).map(Some).map_err(|error| {
 				Error::ResponseFromBinary {
 					binary,
 					error,

lib/src/api/engine/remote/ws/wasm.rs

@@ -405,7 +405,7 @@ impl Response {
 				trace!(target: LOG, "Received an unexpected text message; {text}");
 				Ok(None)
 			}
-			Message::Binary(binary) => msgpack::from_slice(&binary).map(Some).map_err(|error| {
+			Message::Binary(binary) => bung::from_slice(&binary).map(Some).map_err(|error| {
 				Error::ResponseFromBinary {
 					binary,
 					error,

lib/src/api/err/mod.rs

@@ -98,7 +98,7 @@ pub enum Error {
 	#[error("Failed to deserialize a binary response: {error}")]
 	ResponseFromBinary {
 		binary: Vec<u8>,
-		error: msgpack::decode::Error,
+		error: bung::decode::Error,
 	},
 
 	/// Failed to serialize `sql::Value` to JSON string

lib/src/err/mod.rs

@@ -1,5 +1,5 @@
 use crate::sql::idiom::Idiom;
-use msgpack::encode::Error as SerdeError;
+use bung::encode::Error as SerdeError;
 use serde::Serialize;
 use storekey::decode::Error as DecodeError;
 use storekey::encode::Error as EncodeError;

lib/src/sql/value/value.rs

@@ -1856,6 +1856,17 @@ mod tests {
 		assert_eq!(8, std::mem::size_of::<Box<crate::sql::expression::Expression>>());
 	}
 
+	#[test]
+	fn check_serialize() {
+		assert_eq!(1, Value::None.to_vec().len());
+		assert_eq!(1, Value::Null.to_vec().len());
+		assert_eq!(1, Value::True.to_vec().len());
+		assert_eq!(1, Value::False.to_vec().len());
+		assert_eq!(7, Value::from("test").to_vec().len());
+		assert_eq!(17, Value::parse("{ hello: 'world' }").to_vec().len());
+		assert_eq!(24, Value::parse("{ compact: true, schema: 0 }").to_vec().len());
+	}
+
 	#[test]
 	fn serialize_deserialize() {
 		let val = Value::parse(

src/net/import.rs

@@ -45,7 +45,7 @@ async fn handler(
 					"application/cbor" => Ok(output::cbor(&res)),
 					"application/pack" => Ok(output::pack(&res)),
 					// Internal serialization
-					"application/cork" => Ok(output::cork(&res)),
+					"application/bung" => Ok(output::full(&res)),
 					// Return nothing
 					"application/octet-stream" => Ok(output::none()),
 					// An incorrect content-type was requested

src/net/key.rs

@@ -156,7 +156,7 @@ async fn select_all(
 			"application/cbor" => Ok(output::cbor(&res)),
 			"application/pack" => Ok(output::pack(&res)),
 			// Internal serialization
-			"application/cork" => Ok(output::cork(&res)),
+			"application/bung" => Ok(output::full(&res)),
 			// An incorrect content-type was requested
 			_ => Err(warp::reject::custom(Error::InvalidType)),
 		},
@@ -197,7 +197,7 @@ async fn create_all(
 					"application/cbor" => Ok(output::cbor(&res)),
 					"application/pack" => Ok(output::pack(&res)),
 					// Internal serialization
-					"application/cork" => Ok(output::cork(&res)),
+					"application/bung" => Ok(output::full(&res)),
 					// An incorrect content-type was requested
 					_ => Err(warp::reject::custom(Error::InvalidType)),
 				},
@@ -234,7 +234,7 @@ async fn delete_all(
 			"application/cbor" => Ok(output::cbor(&res)),
 			"application/pack" => Ok(output::pack(&res)),
 			// Internal serialization
-			"application/cork" => Ok(output::cork(&res)),
+			"application/bung" => Ok(output::full(&res)),
 			// An incorrect content-type was requested
 			_ => Err(warp::reject::custom(Error::InvalidType)),
 		},
@@ -277,7 +277,7 @@ async fn select_one(
 			"application/cbor" => Ok(output::cbor(&res)),
 			"application/pack" => Ok(output::pack(&res)),
 			// Internal serialization
-			"application/cork" => Ok(output::cork(&res)),
+			"application/bung" => Ok(output::full(&res)),
 			// An incorrect content-type was requested
 			_ => Err(warp::reject::custom(Error::InvalidType)),
 		},
@@ -325,7 +325,7 @@ async fn create_one(
 					"application/cbor" => Ok(output::cbor(&res)),
 					"application/pack" => Ok(output::pack(&res)),
 					// Internal serialization
-					"application/cork" => Ok(output::cork(&res)),
+					"application/bung" => Ok(output::full(&res)),
 					// An incorrect content-type was requested
 					_ => Err(warp::reject::custom(Error::InvalidType)),
 				},
@@ -376,7 +376,7 @@ async fn update_one(
 					"application/cbor" => Ok(output::cbor(&res)),
 					"application/pack" => Ok(output::pack(&res)),
 					// Internal serialization
-					"application/cork" => Ok(output::cork(&res)),
+					"application/bung" => Ok(output::full(&res)),
 					// An incorrect content-type was requested
 					_ => Err(warp::reject::custom(Error::InvalidType)),
 				},
@@ -427,7 +427,7 @@ async fn modify_one(
 					"application/cbor" => Ok(output::cbor(&res)),
 					"application/pack" => Ok(output::pack(&res)),
 					// Internal serialization
-					"application/cork" => Ok(output::cork(&res)),
+					"application/bung" => Ok(output::full(&res)),
 					// An incorrect content-type was requested
 					_ => Err(warp::reject::custom(Error::InvalidType)),
 				},
@@ -471,7 +471,7 @@ async fn delete_one(
 			"application/cbor" => Ok(output::cbor(&res)),
 			"application/pack" => Ok(output::pack(&res)),
 			// Internal serialization
-			"application/cork" => Ok(output::cork(&res)),
+			"application/bung" => Ok(output::full(&res)),
 			// An incorrect content-type was requested
 			_ => Err(warp::reject::custom(Error::InvalidType)),
 		},

src/net/output.rs

@@ -10,7 +10,7 @@ pub enum Output {
 	Json(Vec<u8>), // JSON
 	Cbor(Vec<u8>), // CBOR
 	Pack(Vec<u8>), // MessagePack
-	Cork(Vec<u8>), // Full type serialization
+	Full(Vec<u8>), // Full type serialization
 }
 
 pub fn none() -> Output {
@@ -51,12 +51,12 @@ where
 	}
 }
 
-pub fn cork<T>(val: &T) -> Output
+pub fn full<T>(val: &T) -> Output
 where
 	T: Serialize,
 {
-	match serialize_internal(|| serde_pack::to_vec(val)) {
+	match serialize_internal(|| bung::to_vec(val)) {
-		Ok(v) => Output::Cork(v),
+		Ok(v) => Output::Full(v),
 		Err(_) => Output::Fail,
 	}
 }
@@ -88,9 +88,9 @@ impl warp::Reply for Output {
 				res.headers_mut().insert(CONTENT_TYPE, con);
 				res
 			}
-			Output::Cork(v) => {
+			Output::Full(v) => {
 				let mut res = warp::reply::Response::new(v.into());
-				let con = HeaderValue::from_static("application/cork");
+				let con = HeaderValue::from_static("application/bung");
 				res.headers_mut().insert(CONTENT_TYPE, con);
 				res
 			}

src/net/signin.rs

@@ -64,7 +64,7 @@ async fn handler(
 				Some("application/cbor") => Ok(output::cbor(&Success::new(v))),
 				Some("application/pack") => Ok(output::pack(&Success::new(v))),
 				// Internal serialization
-				Some("application/cork") => Ok(output::cork(&Success::new(v))),
+				Some("application/bung") => Ok(output::full(&Success::new(v))),
 				// Text serialization
 				Some("text/plain") => Ok(output::text(v.unwrap_or_default())),
 				// Return nothing

src/net/signup.rs

@@ -64,7 +64,7 @@ async fn handler(
 				Some("application/cbor") => Ok(output::cbor(&Success::new(v))),
 				Some("application/pack") => Ok(output::pack(&Success::new(v))),
 				// Internal serialization
-				Some("application/cork") => Ok(output::cork(&Success::new(v))),
+				Some("application/bung") => Ok(output::full(&Success::new(v))),
 				// Text serialization
 				Some("text/plain") => Ok(output::text(v.unwrap_or_default())),
 				// Return nothing

src/net/sql.rs

@@ -58,7 +58,7 @@ async fn handler(
 			"application/cbor" => Ok(output::cbor(&res)),
 			"application/pack" => Ok(output::pack(&res)),
 			// Internal serialization
-			"application/cork" => Ok(output::cork(&res)),
+			"application/bung" => Ok(output::full(&res)),
 			// An incorrect content-type was requested
 			_ => Err(warp::reject::custom(Error::InvalidType)),
 		},

src/rpc/res.rs

@@ -49,7 +49,7 @@ impl<T: Serialize> Response<T> {
 				let _ = chn.send(res).await;
 			}
 			Output::Full => {
-				let res = serialize_internal(|| serde_pack::to_vec(&self).unwrap());
+				let res = serialize_internal(|| bung::to_vec(&self).unwrap());
 				let res = Message::binary(res);
 				let _ = chn.send(res).await;
 			}