Extract core surrealdb code into separate library

.gitignore

@@ -33,6 +33,7 @@ Temporary Items
 # -----------------------------------
 
 /target/
+/lib/target/
 
 # -----------------------------------
 # Specific

Cargo.toml

@@ -5,82 +5,32 @@ edition = "2021"
 version = "0.0.0"
 authors = ["Tobie Morgan Hitchcock <tobie@surrealdb.com>"]
 
+[workspace]
+members = ["lib"]
+
+[profile.release]
+lto = true
+# strip = true
+opt-level = 3
+panic = 'abort'
+codegen-units = 1
+
 [dependencies]
-argon2 = "0.3.2"
-async-recursion = "1.0.0"
-bincode = "1.3.3"
-byteorder = "1.4.3"
 bytes = "1.1.0"
-clap = "3.0.10"
-dmp = "0.1.1"
-echodb = "0.2.0"
-futures = "0.3.19"
-fuzzy-matcher = "0.3.7"
+chrono = { version = "0.4.19", features = ["serde"] }
+clap = "3.1.1"
+fern = { version = "0.6.0", features = ["colored"] }
+futures = "0.3.21"
 http = "0.2.6"
-hyper = "0.14.16"
+hyper = "0.14.17"
 log = "0.4.14"
-md-5 = "0.10.0"
-nanoid = "0.4.0"
-nom = "7.1.0"
 once_cell = "1.9.0"
-pbkdf2 = "0.10.0"
-rand = "0.8.4"
-regex = "1.5.4"
-scrypt = "0.8.1"
-sha-1 = "0.10.0"
-sha2 = "0.10.1"
-slug = "0.1.4"
+reqwest = { version = "0.11.9", features = ["blocking"] }
+serde = { version = "1.0.136", features = ["derive"] }
+serde_cbor = "0.11.2"
+serde_json = "1.0.79"
+serde_pack = { version = "1.0.0", package = "rmp-serde" }
+surrealdb = { path = "lib" }
 thiserror = "1.0.30"
-url = "2.2.2"
-utf-8 = "0.7.6"
-
-[dependencies.dec]
-version = "1.20.0"
-package = "rust_decimal"
-features = ["maths", "serde-float"]
-
-[dependencies.geo]
-version = "0.18.0"
-features = ["use-serde"]
-
-[dependencies.fern]
-version = "0.6.0"
-features = ["colored"]
-
-[dependencies.uuid]
-version = "0.8.2"
-features = ["serde", "v4"]
-
-[dependencies.warp]
-version = "0.3.2"
-features = ["compression", "websocket"]
-
-[dependencies.tikv]
-version = "0.1.0"
-package = "tikv-client"
-
-[dependencies.tokio]
-version = "1.15.0"
-features = ["macros"]
-
-[dependencies.reqwest]
-version = "0.11.9"
-features = ["blocking"]
-
-[dependencies.chrono]
-version = "0.4.19"
-features = ["serde"]
-
-[dependencies.serde]
-version = "1.0.135"
-features = ["derive"]
-
-[dependencies.serde_cbor]
-version = "0.11.2"
-
-[dependencies.serde_json]
-version = "1.0.78"
-
-[dependencies.serde_pack]
-version = "1.0.0"
-package = "rmp-serde"
+tokio = { version = "1.17.0", features = ["macros"] }
+warp = { version = "0.3.2", features = ["compression", "websocket"] }

lib/Cargo.toml

@@ -0,0 +1,51 @@
+[package]
+name = "surrealdb"
+publish = true
+edition = "2021"
+version = "0.1.0"
+authors = ["Tobie Morgan Hitchcock <tobie@surrealdb.com>"]
+
+[features]
+default = ["parallel", "kv-tikv", "kv-echodb", "kv-yokudb"]
+parallel = ["tokio"]
+kv-tikv = ["tikv"]
+kv-echodb = ["echodb"]
+kv-indxdb = ["indxdb"]
+kv-yokudb = []
+
+[dependencies]
+argon2 = "0.3.4"
+async-recursion = "1.0.0"
+byteorder = "1.4.3"
+chrono = { version = "0.4.19", features = ["serde"] }
+dec = { version = "1.21.0", package = "rust_decimal", features = ["maths", "serde-float"] }
+dmp = "0.1.1"
+echodb = { version = "0.2.0", optional = true }
+futures = "0.3.21"
+fuzzy-matcher = "0.3.7"
+geo = { version = "0.18.0", features = ["use-serde"] }
+indxdb = { version = "0.1.2", optional = true }
+log = "0.4.14"
+md-5 = "0.10.1"
+nanoid = "0.4.0"
+nom = "7.1.0"
+once_cell = "1.9.0"
+pbkdf2 = "0.10.1"
+trice = "0.1.0"
+rand = "0.8.5"
+regex = "1.5.4"
+msgpack = { version = "1.0.0", package = "rmp-serde" }
+scrypt = "0.9.0"
+serde = { version = "1.0.136", features = ["derive"] }
+sha-1 = "0.10.0"
+sha2 = "0.10.2"
+slug = "0.1.4"
+thiserror = "1.0.30"
+tikv = { version = "0.1.0", package = "tikv-client", optional = true }
+tokio = { version = "1.17.0", features = ["rt-multi-thread"], optional = true }
+url = "2.2.2"
+utf-8 = "0.7.6"
+uuid = { version = "0.8.2", features = ["serde", "v4"] }
+
+[dev-dependencies]
+tokio = { version = "1.17.0", features = ["macros"] }

lib/src/cnf/mod.rs

@@ -0,0 +1,8 @@
+// Specifies how many subqueries will be processed recursively before the query fails.
+pub const MAX_RECURSIVE_QUERIES: usize = 16;
+
+// The characters which are supported in server record IDs.
+pub const ID_CHARS: [char; 36] = [
+	'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i',
+	'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+];

lib/src/dbs/dbs.rs

@@ -8,7 +8,6 @@ use crate::err::Error;
 use crate::kvs::Store;
 use crate::sql;
 use crate::sql::query::Query;
-use hyper::body::Sender;
 use std::sync::Arc;
 
 pub async fn execute(
@@ -54,17 +53,3 @@ pub async fn process(
 	opt.db = session.db.map(Arc::new);
 	exe.execute(ctx, opt, ast).await
 }
-
-pub async fn export(kvs: Store, session: Session, sender: Sender) -> Result<(), Error> {
-	// Create a new query options
-	let mut opt = Options::default();
-	// Create a new query executor
-	let mut exe = Executor::new(kvs);
-	// Create a new execution context
-	let ctx = session.context();
-	// Process database export
-	opt.auth = Arc::new(session.au);
-	opt.ns = session.ns.map(Arc::new);
-	opt.db = session.db.map(Arc::new);
-	exe.export(ctx, opt, sender).await
-}

lib/src/dbs/executor.rs

@@ -10,7 +10,6 @@ use crate::sql::query::Query;
 use crate::sql::statement::Statement;
 use crate::sql::value::Value;
 use futures::lock::Mutex;
-use hyper::body::Sender;
 use std::sync::Arc;
 use trice::Instant;
 
@@ -208,6 +207,9 @@ impl Executor {
 				}
 				// Process param definition statements
 				Statement::Set(stm) => {
+					// Create a transaction
+					let loc = self.begin().await;
+					// Process the statement
 					match stm.compute(&ctx, &opt, &self.txn(), None).await {
 						Ok(val) => {
 							let mut new = Context::new(&ctx);
@@ -217,6 +219,9 @@ impl Executor {
 						}
 						_ => break,
 					}
+					// Cancel transaction
+					self.cancel(loc).await;
+					// Return nothing
 					Ok(Value::None)
 				}
 				// Process all other normal statements
@@ -304,66 +309,4 @@ impl Executor {
 		// Return responses
 		Ok(Responses(out))
 	}
-
-	pub async fn export(
-		&mut self,
-		ctx: Runtime,
-		opt: Options,
-		mut chn: Sender,
-	) -> Result<(), Error> {
-		// Start a new transaction
-		let txn = self.kvs.transaction(false, false).await?;
-		// Output OPTIONS
-		chn.send_data(output!("-- ------------------------------")).await?;
-		chn.send_data(output!("-- OPTION")).await?;
-		chn.send_data(output!("-- ------------------------------")).await?;
-		chn.send_data(output!("")).await?;
-		chn.send_data(output!("OPTION IMPORT;")).await?;
-		chn.send_data(output!("")).await?;
-		// Output LOGINS
-		chn.send_data(output!("-- ------------------------------")).await?;
-		chn.send_data(output!("-- LOGINS")).await?;
-		chn.send_data(output!("-- ------------------------------")).await?;
-		chn.send_data(output!("")).await?;
-		// Output TOKENS
-		chn.send_data(output!("-- ------------------------------")).await?;
-		chn.send_data(output!("-- TOKENS")).await?;
-		chn.send_data(output!("-- ------------------------------")).await?;
-		chn.send_data(output!("")).await?;
-		// Output SCOPES
-		chn.send_data(output!("-- ------------------------------")).await?;
-		chn.send_data(output!("-- SCOPES")).await?;
-		chn.send_data(output!("-- ------------------------------")).await?;
-		chn.send_data(output!("")).await?;
-		// Output TABLES
-		for v in 0..1 {
-			chn.send_data(output!("-- ------------------------------")).await?;
-			chn.send_data(output!(format!("-- TABLE: {}", v))).await?;
-			chn.send_data(output!("-- ------------------------------")).await?;
-			chn.send_data(output!("")).await?;
-		}
-		// Start transaction
-		chn.send_data(output!("-- ------------------------------")).await?;
-		chn.send_data(output!("-- TRANSACTION")).await?;
-		chn.send_data(output!("-- ------------------------------")).await?;
-		chn.send_data(output!("")).await?;
-		chn.send_data(output!("BEGIN TRANSACTION;")).await?;
-		chn.send_data(output!("")).await?;
-		// Output TABLE data
-		for v in 0..1 {
-			chn.send_data(output!("-- ------------------------------")).await?;
-			chn.send_data(output!(format!("-- TABLE DATA: {}", v))).await?;
-			chn.send_data(output!("-- ------------------------------")).await?;
-			chn.send_data(output!("")).await?;
-		}
-		// Commit transaction
-		chn.send_data(output!("-- ------------------------------")).await?;
-		chn.send_data(output!("-- TRANSACTION")).await?;
-		chn.send_data(output!("-- ------------------------------")).await?;
-		chn.send_data(output!("")).await?;
-		chn.send_data(output!("COMMIT TRANSACTION;")).await?;
-		chn.send_data(output!("")).await?;
-		// Everything ok
-		Ok(())
-	}
 }

lib/src/dbs/iterate.rs

@@ -4,6 +4,7 @@ use crate::dbs::Options;
 use crate::dbs::Runtime;
 use crate::dbs::Transaction;
 use crate::err::Error;
+use crate::key::thing;
 use crate::sql::array::Array;
 use crate::sql::model::Model;
 use crate::sql::table::Table;
@@ -13,7 +14,8 @@ use async_recursion::async_recursion;
 use nanoid::nanoid;
 
 impl Value {
-	#[async_recursion]
+	#[cfg_attr(feature = "parallel", async_recursion)]
+	#[cfg_attr(not(feature = "parallel"), async_recursion(?Send))]
 	pub async fn iterate(
 		self,
 		ctx: &Runtime,
@@ -33,7 +35,8 @@ impl Value {
 }
 
 impl Array {
-	#[async_recursion]
+	#[cfg_attr(feature = "parallel", async_recursion)]
+	#[cfg_attr(not(feature = "parallel"), async_recursion(?Send))]
 	pub async fn iterate(
 		self,
 		ctx: &Runtime,

lib/src/dbs/iterator.rs

@@ -128,6 +128,22 @@ impl<'a> Iterator<'a> {
 		}
 	}
 
+	#[cfg(not(feature = "parallel"))]
+	async fn iterate(
+		&mut self,
+		ctx: &Runtime,
+		opt: &Options,
+		txn: &Transaction,
+	) -> Result<(), Error> {
+		// Process all prepared values
+		for v in mem::take(&mut self.readies) {
+			v.iterate(ctx, opt, txn, self).await?;
+		}
+		// Everything processed ok
+		Ok(())
+	}
+
+	#[cfg(feature = "parallel")]
 	async fn iterate(
 		&mut self,
 		ctx: &Runtime,
@@ -135,6 +151,15 @@ impl<'a> Iterator<'a> {
 		txn: &Transaction,
 	) -> Result<(), Error> {
 		match self.parallel {
+			// Run statements sequentially
+			false => {
+				// Process all prepared values
+				for v in mem::take(&mut self.readies) {
+					v.iterate(ctx, opt, txn, self).await?;
+				}
+				// Everything processed ok
+				Ok(())
+			}
 			// Run statements in parallel
 			true => {
 				// Use multi producer channel
@@ -154,15 +179,6 @@ impl<'a> Iterator<'a> {
 				// Everything processed ok
 				Ok(())
 			}
-			// Run statements sequentially
-			false => {
-				// Process all prepared values
-				for v in mem::take(&mut self.readies) {
-					v.iterate(ctx, opt, txn, self).await?;
-				}
-				// Everything processed ok
-				Ok(())
-			}
 		}
 	}
 

lib/src/dbs/mod.rs

@@ -1,5 +1,4 @@
 mod auth;
-mod channel;
 mod dbs;
 mod executor;
 mod iterate;
@@ -13,7 +12,6 @@ mod transaction;
 mod variables;
 
 pub use self::auth::*;
-pub use self::channel::*;
 pub use self::dbs::*;
 pub use self::executor::*;
 pub use self::iterator::*;
@@ -25,5 +23,11 @@ pub use self::statement::*;
 pub use self::transaction::*;
 pub use self::variables::*;
 
+#[cfg(feature = "parallel")]
+mod channel;
+
+#[cfg(feature = "parallel")]
+pub use self::channel::*;
+
 #[cfg(test)]
 pub(crate) mod test;

lib/src/err/mod.rs

@@ -0,0 +1,176 @@
+use crate::key::bytes::decode::Error as DecodeError;
+use crate::key::bytes::encode::Error as EncodeError;
+use crate::sql::thing::Thing;
+use crate::sql::value::Value;
+use msgpack::encode::Error as SerdeError;
+use std::time::Duration;
+use thiserror::Error;
+
+#[cfg(feature = "kv-tikv")]
+use tikv::Error as TiKVError;
+
+#[cfg(feature = "kv-echodb")]
+use echodb::err::Error as EchoDBError;
+
+#[cfg(feature = "kv-indxdb")]
+use indxdb::err::Error as IndxDBError;
+
+#[cfg(feature = "parallel")]
+use tokio::sync::mpsc::error::SendError as TokioError;
+
+#[derive(Error, Debug)]
+pub enum Error {
+	#[error("Couldn't setup connection to underlying datastore")]
+	DsError,
+
+	#[error("Couldn't create a database transaction")]
+	TxError,
+
+	#[error("Couldn't update a finished transaction")]
+	TxFinishedError,
+
+	#[error("Couldn't write to a read only transaction")]
+	TxReadonlyError,
+
+	#[error("Specify a namespace to use")]
+	NsError,
+
+	#[error("Specify a database to use")]
+	DbError,
+
+	#[error("Specify some SQL code to execute")]
+	EmptyError,
+
+	#[error("The query failed to complete in time")]
+	TimeoutError,
+
+	#[error("The query was cancelled before completion")]
+	CancelledError,
+
+	#[error("Parse error on line {line} at character {char} when parsing '{sql}'")]
+	ParseError {
+		line: usize,
+		char: usize,
+		sql: String,
+	},
+
+	#[error("The JSON Patch contains invalid operations. {message}")]
+	PatchError {
+		message: String,
+	},
+
+	#[error("Problem with embedded script function. {message}")]
+	LanguageError {
+		message: String,
+	},
+
+	#[error("Incorrect arguments for function {name}(). {message}")]
+	ArgumentsError {
+		name: String,
+		message: String,
+	},
+
+	#[error("Query timeout of {timer:?} exceeded")]
+	QueryTimeoutError {
+		timer: Duration,
+	},
+
+	#[error("Query not executed due to cancelled transaction")]
+	QueryCancelledError,
+
+	#[error("Query not executed due to failed transaction")]
+	QueryExecutionError,
+
+	#[error("You don't have permission to perform this query type")]
+	QueryPermissionsError,
+
+	#[error("You don't have permission to change to the {ns} namespace")]
+	NsAuthenticationError {
+		ns: String,
+	},
+
+	#[error("You don't have permission to change to the {db} database")]
+	DbAuthenticationError {
+		db: String,
+	},
+
+	#[error("Too many recursive subqueries have been set")]
+	RecursiveSubqueryError {
+		limit: usize,
+	},
+
+	#[error("Can not execute CREATE query using value '{value}'")]
+	CreateStatementError {
+		value: Value,
+	},
+
+	#[error("Can not execute UPDATE query using value '{value}'")]
+	UpdateStatementError {
+		value: Value,
+	},
+
+	#[error("Can not execute RELATE query using value '{value}'")]
+	RelateStatementError {
+		value: Value,
+	},
+
+	#[error("Can not execute DELETE query using value '{value}'")]
+	DeleteStatementError {
+		value: Value,
+	},
+
+	#[error("Can not execute INSERT query using value '{value}'")]
+	InsertStatementError {
+		value: Value,
+	},
+
+	#[error("You don't have permission to run the `{query}` query on the `{table}` table")]
+	TablePermissionsError {
+		query: String,
+		table: String,
+	},
+
+	#[error("Unable to write to the `{table}` table while setup as a view")]
+	TableViewError {
+		table: String,
+	},
+
+	#[error("Database record `{thing}` already exists")]
+	RecordExistsError {
+		thing: Thing,
+	},
+
+	#[error("Database index `{index}` already contains `{thing}`")]
+	RecordIndexError {
+		index: String,
+		thing: Thing,
+	},
+
+	#[error("Conditional clause is not truthy")]
+	IgnoreError,
+
+	#[error("Serde error: {0}")]
+	SerdeError(#[from] SerdeError),
+
+	#[error("Key encoding error: {0}")]
+	EncodeError(#[from] EncodeError),
+
+	#[error("Key decoding error: {0}")]
+	DecodeError(#[from] DecodeError),
+
+	#[cfg(feature = "kv-echodb")]
+	#[error("Datastore error: {0}")]
+	EchoDBError(#[from] EchoDBError),
+
+	#[cfg(feature = "kv-indxdb")]
+	#[error("Datastore error: {0}")]
+	IndxDBError(#[from] IndxDBError),
+
+	#[cfg(feature = "kv-tikv")]
+	#[error("Datastore error: {0}")]
+	TiKVError(#[from] TiKVError),
+
+	#[cfg(feature = "parallel")]
+	#[error("Tokio Error: {0}")]
+	TokioError(#[from] TokioError<(Option<Thing>, Value)>),
+}