feat: WITH clause on SELECT statement (#2304)

lib/src/dbs/processor.rs

@@ -550,49 +550,50 @@ impl<'a> Processor<'a> {
 		txn.lock().await.check_ns_db_tb(opt.ns(), opt.db(), &table.0, opt.strict).await?;
 		if let Some(pla) = ctx.get_query_planner() {
 			if let Some(exe) = pla.get_query_executor(&table.0) {
-				let mut iterator = exe.new_iterator(opt, ir, io).await?;
+				if let Some(mut iterator) = exe.new_iterator(opt, ir, io).await? {
-				let mut things = iterator.next_batch(txn, 1000).await?;
+					let mut things = iterator.next_batch(txn, 1000).await?;
-				while !things.is_empty() {
+					while !things.is_empty() {
-					// Check if the context is finished
+						// Check if the context is finished
-					if ctx.is_done() {
-						break;
-					}
-
-					for (thing, doc_id) in things {
-						// Check the context
 						if ctx.is_done() {
 							break;
 						}
 
-						// If the record is from another table we can skip
+						for (thing, doc_id) in things {
-						if !thing.tb.eq(table.as_str()) {
+							// Check the context
-							continue;
+							if ctx.is_done() {
+								break;
+							}
+
+							// If the record is from another table we can skip
+							if !thing.tb.eq(table.as_str()) {
+								continue;
+							}
+
+							// Fetch the data from the store
+							let key = thing::new(opt.ns(), opt.db(), &table.0, &thing.id);
+							let val = txn.lock().await.get(key.clone()).await?;
+							let rid = Thing::from((key.tb, key.id));
+							// Parse the data from the store
+							let val = Operable::Value(match val {
+								Some(v) => Value::from(v),
+								None => Value::None,
+							});
+							// Process the document record
+							let pro = Processed {
+								ir: Some(ir),
+								rid: Some(rid),
+								doc_id: Some(doc_id),
+								val,
+							};
+							self.process(ctx, opt, txn, stm, pro).await?;
 						}
 
-						// Fetch the data from the store
+						// Collect the next batch of ids
-						let key = thing::new(opt.ns(), opt.db(), &table.0, &thing.id);
+						things = iterator.next_batch(txn, 1000).await?;
-						let val = txn.lock().await.get(key.clone()).await?;
-						let rid = Thing::from((key.tb, key.id));
-						// Parse the data from the store
-						let val = Operable::Value(match val {
-							Some(v) => Value::from(v),
-							None => Value::None,
-						});
-						// Process the document record
-						let pro = Processed {
-							ir: Some(ir),
-							rid: Some(rid),
-							doc_id: Some(doc_id),
-							val,
-						};
-						self.process(ctx, opt, txn, stm, pro).await?;
 					}
-
+					// Everything ok
-					// Collect the next batch of ids
+					return Ok(());
-					things = iterator.next_batch(txn, 1000).await?;
 				}
-				// Everything ok
-				return Ok(());
 			}
 		}
 		Err(Error::QueryNotExecutedDetail {

lib/src/err/mod.rs

@@ -507,10 +507,6 @@ pub enum Error {
 		feature: &'static str,
 	},
 
-	#[doc(hidden)]
-	#[error("Bypass the query planner")]
-	BypassQueryPlanner,
-
 	/// Duplicated match references are not allowed
 	#[error("Duplicated Match reference: {mr}")]
 	DuplicatedMatchRef {

lib/src/idx/planner/executor.rs

@@ -121,7 +121,7 @@ impl QueryExecutor {
 		opt: &Options,
 		ir: IteratorRef,
 		io: IndexOption,
-	) -> Result<ThingIterator, Error> {
+	) -> Result<Option<ThingIterator>, Error> {
 		match &io.ix().index {
 			Index::Idx => Self::new_index_iterator(opt, io),
 			Index::Uniq => Self::new_unique_index_iterator(opt, io),
@@ -131,45 +131,48 @@ impl QueryExecutor {
 		}
 	}
 
-	fn new_index_iterator(opt: &Options, io: IndexOption) -> Result<ThingIterator, Error> {
+	fn new_index_iterator(opt: &Options, io: IndexOption) -> Result<Option<ThingIterator>, Error> {
 		if io.op() == &Operator::Equal {
-			return Ok(ThingIterator::NonUniqueEqual(NonUniqueEqualThingIterator::new(
+			return Ok(Some(ThingIterator::NonUniqueEqual(NonUniqueEqualThingIterator::new(
 				opt,
 				io.ix(),
 				io.value(),
-			)?));
+			)?)));
 		}
-		Err(Error::BypassQueryPlanner)
+		Ok(None)
 	}
 
-	fn new_unique_index_iterator(opt: &Options, io: IndexOption) -> Result<ThingIterator, Error> {
+	fn new_unique_index_iterator(
+		opt: &Options,
+		io: IndexOption,
+	) -> Result<Option<ThingIterator>, Error> {
 		if io.op() == &Operator::Equal {
-			return Ok(ThingIterator::UniqueEqual(UniqueEqualThingIterator::new(
+			return Ok(Some(ThingIterator::UniqueEqual(UniqueEqualThingIterator::new(
 				opt,
 				io.ix(),
 				io.value(),
-			)?));
+			)?)));
 		}
-		Err(Error::BypassQueryPlanner)
+		Ok(None)
 	}
 
 	async fn new_search_index_iterator(
 		&self,
 		ir: IteratorRef,
 		io: IndexOption,
-	) -> Result<ThingIterator, Error> {
+	) -> Result<Option<ThingIterator>, Error> {
 		if let Some(exp) = self.iterators.get(ir as usize) {
 			if let Operator::Matches(_) = io.op() {
 				let ixn = &io.ix().name.0;
 				if let Some(fti) = self.ft_map.get(ixn) {
 					if let Some(fte) = self.exp_entries.get(exp) {
 						let it = MatchesThingIterator::new(fti, fte.0.terms_docs.clone()).await?;
-						return Ok(ThingIterator::Matches(it));
+						return Ok(Some(ThingIterator::Matches(it)));
 					}
 				}
 			}
 		}
-		Err(Error::BypassQueryPlanner)
+		Ok(None)
 	}
 
 	pub(crate) async fn matches(

lib/src/idx/planner/mod.rs

@@ -9,11 +9,13 @@ use crate::err::Error;
 use crate::idx::planner::executor::QueryExecutor;
 use crate::idx::planner::plan::{Plan, PlanBuilder};
 use crate::idx::planner::tree::Tree;
+use crate::sql::with::With;
 use crate::sql::{Cond, Table};
 use std::collections::HashMap;
 
 pub(crate) struct QueryPlanner<'a> {
 	opt: &'a Options,
+	with: &'a Option<With>,
 	cond: &'a Option<Cond>,
 	/// There is one executor per table
 	executors: HashMap<String, QueryExecutor>,
@@ -21,9 +23,10 @@ pub(crate) struct QueryPlanner<'a> {
 }
 
 impl<'a> QueryPlanner<'a> {
-	pub(crate) fn new(opt: &'a Options, cond: &'a Option<Cond>) -> Self {
+	pub(crate) fn new(opt: &'a Options, with: &'a Option<With>, cond: &'a Option<Cond>) -> Self {
 		Self {
 			opt,
+			with,
 			cond,
 			executors: HashMap::default(),
 			requires_distinct: false,
@@ -40,24 +43,21 @@ impl<'a> QueryPlanner<'a> {
 		let res = Tree::build(ctx, self.opt, txn, &t, self.cond).await?;
 		if let Some((node, im)) = res {
 			let mut exe = QueryExecutor::new(self.opt, txn, &t, im).await?;
-			let ok = match PlanBuilder::build(node) {
+			let ok = match PlanBuilder::build(node, self.with)? {
-				Ok(plan) => match plan {
+				Plan::SingleIndex(exp, io) => {
-					Plan::SingleIndex(exp, io) => {
+					let ir = exe.add_iterator(exp);
+					it.ingest(Iterable::Index(t.clone(), ir, io));
+					true
+				}
+				Plan::MultiIndex(v) => {
+					for (exp, io) in v {
 						let ir = exe.add_iterator(exp);
 						it.ingest(Iterable::Index(t.clone(), ir, io));
-						true
+						self.requires_distinct = true;
 					}
-					Plan::MultiIndex(v) => {
+					true
-						for (exp, io) in v {
+				}
-							let ir = exe.add_iterator(exp);
+				Plan::TableIterator => false,
-							it.ingest(Iterable::Index(t.clone(), ir, io));
-							self.requires_distinct = true;
-						}
-						true
-					}
-				},
-				Err(Error::BypassQueryPlanner) => false,
-				Err(e) => return Err(e),
 			};
 			self.executors.insert(t.0.clone(), exe);
 			if ok {

lib/src/idx/planner/plan.rs

@@ -2,30 +2,40 @@ use crate::err::Error;
 use crate::idx::ft::MatchRef;
 use crate::idx::planner::tree::Node;
 use crate::sql::statements::DefineIndexStatement;
+use crate::sql::with::With;
 use crate::sql::Object;
 use crate::sql::{Expression, Idiom, Operator, Value};
 use std::collections::HashMap;
 use std::hash::Hash;
 use std::sync::Arc;
 
-pub(super) struct PlanBuilder {
+pub(super) struct PlanBuilder<'a> {
 	indexes: Vec<(Expression, IndexOption)>,
+	with: &'a Option<With>,
 	all_and: bool,
 	all_exp_with_index: bool,
 }
 
-impl PlanBuilder {
+impl<'a> PlanBuilder<'a> {
-	pub(super) fn build(root: Node) -> Result<Plan, Error> {
+	pub(super) fn build(root: Node, with: &'a Option<With>) -> Result<Plan, Error> {
+		if let Some(with) = with {
+			if matches!(with, With::NoIndex) {
+				return Ok(Plan::TableIterator);
+			}
+		}
 		let mut b = PlanBuilder {
 			indexes: Vec::new(),
+			with,
 			all_and: true,
 			all_exp_with_index: true,
 		};
 		// Browse the AST and collect information
-		b.eval_node(root)?;
+		if !b.eval_node(root)? {
+			return Ok(Plan::TableIterator);
+		}
 		// If we didn't found any index, we're done with no index plan
 		if b.indexes.is_empty() {
-			return Err(Error::BypassQueryPlanner);
+			return Ok(Plan::TableIterator);
 		}
 		// If every boolean operator are AND then we can use the single index plan
 		if b.all_and {
@@ -37,10 +47,22 @@ impl PlanBuilder {
 		if b.all_exp_with_index {
 			return Ok(Plan::MultiIndex(b.indexes));
 		}
-		Err(Error::BypassQueryPlanner)
+		Ok(Plan::TableIterator)
 	}
 
-	fn eval_node(&mut self, node: Node) -> Result<(), Error> {
+	// Check if we have an explicit list of index we can use
+	fn filter_index_option(&self, io: Option<IndexOption>) -> Option<IndexOption> {
+		if let Some(io) = &io {
+			if let Some(With::Index(ixs)) = self.with {
+				if !ixs.contains(&io.ix().name.0) {
+					return None;
+				}
+			}
+		}
+		io
+	}
+
+	fn eval_node(&mut self, node: Node) -> Result<bool, Error> {
 		match node {
 			Node::Expression {
 				io,
@@ -52,15 +74,15 @@ impl PlanBuilder {
 					self.all_and = false;
 				}
 				let is_bool = self.check_boolean_operator(exp.operator());
-				if let Some(io) = io {
+				if let Some(io) = self.filter_index_option(io) {
 					self.add_index_option(exp, io);
 				} else if self.all_exp_with_index && !is_bool {
 					self.all_exp_with_index = false;
 				}
 				self.eval_expression(*left, *right)
 			}
-			Node::Unsupported => Err(Error::BypassQueryPlanner),
+			Node::Unsupported => Ok(false),
-			_ => Ok(()),
+			_ => Ok(true),
 		}
 	}
 
@@ -77,10 +99,14 @@ impl PlanBuilder {
 		}
 	}
 
-	fn eval_expression(&mut self, left: Node, right: Node) -> Result<(), Error> {
+	fn eval_expression(&mut self, left: Node, right: Node) -> Result<bool, Error> {
-		self.eval_node(left)?;
+		if !self.eval_node(left)? {
-		self.eval_node(right)?;
+			return Ok(false);
-		Ok(())
+		}
+		if !self.eval_node(right)? {
+			return Ok(false);
+		}
+		Ok(true)
 	}
 
 	fn add_index_option(&mut self, e: Expression, i: IndexOption) {
@@ -89,6 +115,7 @@ impl PlanBuilder {
 }
 
 pub(super) enum Plan {
+	TableIterator,
 	SingleIndex(Expression, IndexOption),
 	MultiIndex(Vec<(Expression, IndexOption)>),
 }

lib/src/idx/planner/tree.rs

@@ -19,7 +19,7 @@ impl Tree {
 		opt: &'a Options,
 		txn: &'a Transaction,
 		table: &'a Table,
-		cond: &Option<Cond>,
+		cond: &'a Option<Cond>,
 	) -> Result<Option<(Node, IndexMap)>, Error> {
 		let mut b = TreeBuilder {
 			ctx,

lib/src/sql/mod.rs

@@ -68,6 +68,7 @@ pub(crate) mod uuid;
 pub(crate) mod value;
 pub(crate) mod version;
 pub(crate) mod view;
+pub(crate) mod with;
 
 #[cfg(test)]
 pub(crate) mod test;

lib/src/sql/statements/select.rs

@@ -24,6 +24,7 @@ use crate::sql::start::{start, Start};
 use crate::sql::timeout::{timeout, Timeout};
 use crate::sql::value::{selects, Value, Values};
 use crate::sql::version::{version, Version};
+use crate::sql::with::{with, With};
 use derive::Store;
 use nom::bytes::complete::tag_no_case;
 use nom::combinator::opt;
@@ -35,6 +36,7 @@ use std::fmt;
 pub struct SelectStatement {
 	pub expr: Fields,
 	pub what: Values,
+	pub with: Option<With>,
 	pub cond: Option<Cond>,
 	pub split: Option<Splits>,
 	pub group: Option<Groups>,
@@ -91,7 +93,7 @@ impl SelectStatement {
 		let opt = &opt.new_with_futures(false);
 
 		// Get a query planner
-		let mut planner = QueryPlanner::new(opt, &self.cond);
+		let mut planner = QueryPlanner::new(opt, &self.with, &self.cond);
 		// Loop over the select targets
 		for w in self.what.0.iter() {
 			let v = w.compute(ctx, opt, txn, doc).await?;
@@ -143,6 +145,9 @@ impl SelectStatement {
 impl fmt::Display for SelectStatement {
 	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 		write!(f, "SELECT {} FROM {}", self.expr, self.what)?;
+		if let Some(ref v) = self.with {
+			write!(f, " {v}")?
+		}
 		if let Some(ref v) = self.cond {
 			write!(f, " {v}")?
 		}
@@ -188,6 +193,7 @@ pub fn select(i: &str) -> IResult<&str, SelectStatement> {
 	let (i, _) = tag_no_case("FROM")(i)?;
 	let (i, _) = shouldbespace(i)?;
 	let (i, what) = selects(i)?;
+	let (i, with) = opt(preceded(shouldbespace, with))(i)?;
 	let (i, cond) = opt(preceded(shouldbespace, cond))(i)?;
 	let (i, split) = opt(preceded(shouldbespace, split))(i)?;
 	check_split_on_fields(i, &expr, &split)?;
@@ -207,6 +213,7 @@ pub fn select(i: &str) -> IResult<&str, SelectStatement> {
 		SelectStatement {
 			expr,
 			what,
+			with,
 			cond,
 			split,
 			group,

lib/src/sql/value/serde/ser/mod.rs

@@ -39,6 +39,7 @@ mod timeout;
 mod uuid;
 mod value;
 mod version;
+mod with;
 
 use serde::ser::Error;
 use serde::ser::Serialize;

lib/src/sql/value/serde/ser/statement/select.rs

@@ -2,6 +2,7 @@ use crate::err::Error;
 use crate::sql::explain::Explain;
 use crate::sql::statements::SelectStatement;
 use crate::sql::value::serde::ser;
+use crate::sql::with::With;
 use crate::sql::Cond;
 use crate::sql::Fetchs;
 use crate::sql::Fields;
@@ -48,6 +49,7 @@ impl ser::Serializer for Serializer {
 pub struct SerializeSelectStatement {
 	expr: Option<Fields>,
 	what: Option<Values>,
+	with: Option<With>,
 	cond: Option<Cond>,
 	split: Option<Splits>,
 	group: Option<Groups>,
@@ -76,6 +78,9 @@ impl serde::ser::SerializeStruct for SerializeSelectStatement {
 			"what" => {
 				self.what = Some(Values(value.serialize(ser::value::vec::Serializer.wrap())?));
 			}
+			"with" => {
+				self.with = value.serialize(ser::with::opt::Serializer.wrap())?;
+			}
 			"cond" => {
 				self.cond = value.serialize(ser::cond::opt::Serializer.wrap())?;
 			}
@@ -121,6 +126,7 @@ impl serde::ser::SerializeStruct for SerializeSelectStatement {
 			(Some(expr), Some(what), Some(parallel)) => Ok(SelectStatement {
 				expr,
 				what,
+				with: self.with,
 				parallel,
 				explain: self.explain,
 				cond: self.cond,
@@ -248,4 +254,34 @@ mod tests {
 		let value: SelectStatement = stmt.serialize(Serializer.wrap()).unwrap();
 		assert_eq!(value, stmt);
 	}
+
+	#[test]
+	fn with_explain_full() {
+		let stmt = SelectStatement {
+			explain: Some(Explain(true)),
+			..Default::default()
+		};
+		let value: SelectStatement = stmt.serialize(Serializer.wrap()).unwrap();
+		assert_eq!(value, stmt);
+	}
+
+	#[test]
+	fn with_with_noindex() {
+		let stmt = SelectStatement {
+			with: Some(With::NoIndex),
+			..Default::default()
+		};
+		let value: SelectStatement = stmt.serialize(Serializer.wrap()).unwrap();
+		assert_eq!(value, stmt);
+	}
+
+	#[test]
+	fn with_with_index() {
+		let stmt = SelectStatement {
+			with: Some(With::Index(vec!["uniq".to_string(), "ft".to_string(), "idx".to_string()])),
+			..Default::default()
+		};
+		let value: SelectStatement = stmt.serialize(Serializer.wrap()).unwrap();
+		assert_eq!(value, stmt);
+	}
 }

lib/src/sql/value/serde/ser/with/mod.rs

@@ -0,0 +1,78 @@
+pub(super) mod opt;
+
+use crate::err::Error;
+use crate::sql::value::serde::ser;
+use crate::sql::with::With;
+use serde::ser::Error as _;
+use serde::ser::Impossible;
+use serde::ser::Serialize;
+
+pub(super) struct Serializer;
+
+impl ser::Serializer for Serializer {
+	type Ok = With;
+	type Error = Error;
+
+	type SerializeSeq = Impossible<With, Error>;
+	type SerializeTuple = Impossible<With, Error>;
+	type SerializeTupleStruct = Impossible<With, Error>;
+	type SerializeTupleVariant = Impossible<With, Error>;
+	type SerializeMap = Impossible<With, Error>;
+	type SerializeStruct = Impossible<With, Error>;
+	type SerializeStructVariant = Impossible<With, Error>;
+
+	const EXPECTED: &'static str = "an enum `With`";
+
+	#[inline]
+	fn serialize_unit_variant(
+		self,
+		name: &'static str,
+		_variant_index: u32,
+		variant: &'static str,
+	) -> Result<Self::Ok, Error> {
+		match variant {
+			"NoIndex" => Ok(With::NoIndex),
+			variant => Err(Error::custom(format!("unexpected unit variant `{name}::{variant}`"))),
+		}
+	}
+
+	#[inline]
+	fn serialize_newtype_variant<T>(
+		self,
+		name: &'static str,
+		_variant_index: u32,
+		variant: &'static str,
+		value: &T,
+	) -> Result<Self::Ok, Error>
+	where
+		T: ?Sized + Serialize,
+	{
+		match variant {
+			"Index" => Ok(With::Index(value.serialize(ser::string::vec::Serializer.wrap())?)),
+			variant => {
+				Err(Error::custom(format!("unexpected newtype variant `{name}::{variant}`")))
+			}
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use ser::Serializer as _;
+	use serde::Serialize;
+
+	#[test]
+	fn with_noindex() {
+		let with = With::NoIndex;
+		let serialized = with.serialize(Serializer.wrap()).unwrap();
+		assert_eq!(with, serialized);
+	}
+
+	#[test]
+	fn with_index() {
+		let with = With::Index(vec!["idx".to_string(), "uniq".to_string()]);
+		let serialized = with.serialize(Serializer.wrap()).unwrap();
+		assert_eq!(with, serialized);
+	}
+}

lib/src/sql/value/serde/ser/with/opt.rs

@@ -0,0 +1,55 @@
+use crate::err::Error;
+use crate::sql::value::serde::ser;
+use crate::sql::with::With;
+use serde::ser::Impossible;
+use serde::ser::Serialize;
+
+pub struct Serializer;
+
+impl ser::Serializer for Serializer {
+	type Ok = Option<With>;
+	type Error = Error;
+
+	type SerializeSeq = Impossible<Option<With>, Error>;
+	type SerializeTuple = Impossible<Option<With>, Error>;
+	type SerializeTupleStruct = Impossible<Option<With>, Error>;
+	type SerializeTupleVariant = Impossible<Option<With>, Error>;
+	type SerializeMap = Impossible<Option<With>, Error>;
+	type SerializeStruct = Impossible<Option<With>, Error>;
+	type SerializeStructVariant = Impossible<Option<With>, Error>;
+
+	const EXPECTED: &'static str = "an `Option<With>`";
+
+	#[inline]
+	fn serialize_none(self) -> Result<Self::Ok, Self::Error> {
+		Ok(None)
+	}
+
+	#[inline]
+	fn serialize_some<T>(self, value: &T) -> Result<Self::Ok, Self::Error>
+	where
+		T: ?Sized + Serialize,
+	{
+		Ok(Some(value.serialize(ser::with::Serializer.wrap())?))
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use ser::Serializer as _;
+
+	#[test]
+	fn none() {
+		let option: Option<With> = None;
+		let serialized = option.serialize(Serializer.wrap()).unwrap();
+		assert_eq!(option, serialized);
+	}
+
+	#[test]
+	fn some() {
+		let option = Some(With::NoIndex);
+		let serialized = option.serialize(Serializer.wrap()).unwrap();
+		assert_eq!(option, serialized);
+	}
+}

lib/src/sql/with.rs

@@ -0,0 +1,73 @@
+use crate::sql::comment::shouldbespace;
+use crate::sql::common::commas;
+use crate::sql::error::IResult;
+use crate::sql::ident::ident_raw;
+use derive::Store;
+use nom::branch::alt;
+use nom::bytes::complete::tag_no_case;
+use nom::multi::separated_list1;
+use serde::{Deserialize, Serialize};
+use std::fmt::{Display, Formatter, Result};
+
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Store, Hash)]
+pub enum With {
+	NoIndex,
+	Index(Vec<String>),
+}
+
+impl Display for With {
+	fn fmt(&self, f: &mut Formatter) -> Result {
+		f.write_str("WITH")?;
+		match self {
+			With::NoIndex => f.write_str(" NOINDEX"),
+			With::Index(i) => {
+				f.write_str(" INDEX ")?;
+				f.write_str(&i.join(","))
+			}
+		}
+	}
+}
+
+fn no_index(i: &str) -> IResult<&str, With> {
+	let (i, _) = tag_no_case("NOINDEX")(i)?;
+	Ok((i, With::NoIndex))
+}
+
+fn index(i: &str) -> IResult<&str, With> {
+	let (i, _) = tag_no_case("INDEX")(i)?;
+	let (i, _) = shouldbespace(i)?;
+	let (i, v) = separated_list1(commas, ident_raw)(i)?;
+	Ok((i, With::Index(v)))
+}
+
+pub fn with(i: &str) -> IResult<&str, With> {
+	let (i, _) = tag_no_case("WITH")(i)?;
+	let (i, _) = shouldbespace(i)?;
+	alt((no_index, index))(i)
+}
+
+#[cfg(test)]
+mod tests {
+
+	use super::*;
+
+	#[test]
+	fn with_no_index() {
+		let sql = "WITH NOINDEX";
+		let res = with(sql);
+		assert!(res.is_ok());
+		let out = res.unwrap().1;
+		assert_eq!(out, With::NoIndex);
+		assert_eq!("WITH NOINDEX", format!("{}", out));
+	}
+
+	#[test]
+	fn with_index() {
+		let sql = "WITH INDEX idx,uniq";
+		let res = with(sql);
+		assert!(res.is_ok());
+		let out = res.unwrap().1;
+		assert_eq!(out, With::Index(vec!["idx".to_string(), "uniq".to_string()]));
+		assert_eq!("WITH INDEX idx,uniq", format!("{}", out));
+	}
+}

lib/tests/planner.rs

@@ -1,19 +1,157 @@
 mod parse;
 
 use parse::Parse;
-use surrealdb::dbs::Session;
+use surrealdb::dbs::{Response, Session};
 use surrealdb::err::Error;
 use surrealdb::kvs::Datastore;
 use surrealdb::sql::Value;
 
-async fn test_select_where_iterate_multi_index(parallel: bool) -> Result<(), Error> {
+#[tokio::test]
-	let parallel = if parallel {
+async fn select_where_iterate_three_multi_index() -> Result<(), Error> {
-		"PARALLEL"
+	let mut res = execute_test(&three_multi_index_query("", ""), 12).await?;
-	} else {
+	check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Tobie' }, { name: 'Lizzie' }]")?;
-		""
+	// OR results
-	};
+	check_result(&mut res, THREE_MULTI_INDEX_EXPLAIN)?;
-	let sql = format!(
+	// AND results
-		"
+	check_result(&mut res, "[{name: 'Jaime'}]")?;
+	check_result(&mut res, SINGLE_INDEX_FT_EXPLAIN)?;
+	Ok(())
+}
+
+#[tokio::test]
+async fn select_where_iterate_three_multi_index_parallel() -> Result<(), Error> {
+	let mut res = execute_test(&three_multi_index_query("", "PARALLEL"), 12).await?;
+	// OR results
+	check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Tobie' }, { name: 'Lizzie' }]")?;
+	check_result(&mut res, THREE_MULTI_INDEX_EXPLAIN)?;
+	// AND results
+	check_result(&mut res, "[{name: 'Jaime'}]")?;
+	check_result(&mut res, SINGLE_INDEX_FT_EXPLAIN)?;
+	Ok(())
+}
+
+#[tokio::test]
+async fn select_where_iterate_three_multi_index_with_all_index() -> Result<(), Error> {
+	let mut res =
+		execute_test(&three_multi_index_query("WITH INDEX uniq_name,idx_genre,ft_company", ""), 12)
+			.await?;
+	// OR results
+	check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Tobie' }, { name: 'Lizzie' }]")?;
+	check_result(&mut res, THREE_MULTI_INDEX_EXPLAIN)?;
+	// AND results
+	check_result(&mut res, "[{name: 'Jaime'}]")?;
+	check_result(&mut res, SINGLE_INDEX_FT_EXPLAIN)?;
+	Ok(())
+}
+
+#[tokio::test]
+async fn select_where_iterate_three_multi_index_with_one_ft_index() -> Result<(), Error> {
+	let mut res = execute_test(&three_multi_index_query("WITH INDEX ft_company", ""), 12).await?;
+	// OR results
+	check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Lizzie' }, { name: 'Tobie' } ]")?;
+	check_result(&mut res, THREE_TABLE_EXPLAIN)?;
+	// AND results
+	check_result(&mut res, "[{name: 'Jaime'}]")?;
+	check_result(&mut res, SINGLE_INDEX_FT_EXPLAIN)?;
+	Ok(())
+}
+
+#[tokio::test]
+async fn select_where_iterate_three_multi_index_with_one_index() -> Result<(), Error> {
+	let mut res = execute_test(&three_multi_index_query("WITH INDEX uniq_name", ""), 12).await?;
+	// OR results
+	check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Lizzie' }, { name: 'Tobie' } ]")?;
+	check_result(&mut res, THREE_TABLE_EXPLAIN)?;
+	// AND results
+	check_result(&mut res, "[{name: 'Jaime'}]")?;
+	check_result(&mut res, SINGLE_INDEX_UNIQ_EXPLAIN)?;
+	Ok(())
+}
+
+#[tokio::test]
+async fn select_where_iterate_two_multi_index() -> Result<(), Error> {
+	let mut res = execute_test(&two_multi_index_query("", ""), 9).await?;
+	// OR results
+	check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Tobie' }]")?;
+	check_result(&mut res, TWO_MULTI_INDEX_EXPLAIN)?;
+	// AND results
+	check_result(&mut res, "[{name: 'Jaime'}]")?;
+	check_result(&mut res, SINGLE_INDEX_IDX_EXPLAIN)?;
+	Ok(())
+}
+
+#[tokio::test]
+async fn select_where_iterate_two_multi_index_with_one_index() -> Result<(), Error> {
+	let mut res = execute_test(&two_multi_index_query("WITH INDEX idx_genre", ""), 9).await?;
+	// OR results
+	check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Tobie' }]")?;
+	check_result(&mut res, &table_explain(2))?;
+	// AND results
+	check_result(&mut res, "[{name: 'Jaime'}]")?;
+	check_result(&mut res, SINGLE_INDEX_IDX_EXPLAIN)?;
+	Ok(())
+}
+
+#[tokio::test]
+async fn select_where_iterate_two_multi_index_with_two_index() -> Result<(), Error> {
+	let mut res =
+		execute_test(&two_multi_index_query("WITH INDEX idx_genre,uniq_name", ""), 9).await?;
+	// OR results
+	check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Tobie' }]")?;
+	check_result(&mut res, TWO_MULTI_INDEX_EXPLAIN)?;
+	// AND results
+	check_result(&mut res, "[{name: 'Jaime'}]")?;
+	check_result(&mut res, SINGLE_INDEX_IDX_EXPLAIN)?;
+	Ok(())
+}
+
+#[tokio::test]
+async fn select_where_iterate_two_no_index() -> Result<(), Error> {
+	let mut res = execute_test(&two_multi_index_query("WITH NOINDEX", ""), 9).await?;
+	// OR results
+	check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Tobie' }]")?;
+	check_result(&mut res, &table_explain(2))?;
+	// AND results
+	check_result(&mut res, "[{name: 'Jaime'}]")?;
+	check_result(&mut res, &table_explain(1))?;
+	Ok(())
+}
+
+async fn execute_test(sql: &str, expected_result: usize) -> Result<Vec<Response>, Error> {
+	let dbs = Datastore::new("memory").await?;
+	let ses = Session::for_kv().with_ns("test").with_db("test");
+	let mut res = dbs.execute(sql, &ses, None).await?;
+	assert_eq!(res.len(), expected_result);
+	// Check that the setup is ok
+	for _ in 0..(expected_result - 4) {
+		let _ = res.remove(0).result?;
+	}
+	Ok(res)
+}
+
+fn check_result(res: &mut Vec<Response>, expected: &str) -> Result<(), Error> {
+	let tmp = res.remove(0).result?;
+	let val = Value::parse(expected);
+	assert_eq!(format!("{:#}", tmp), format!("{:#}", val));
+	Ok(())
+}
+
+fn two_multi_index_query(with: &str, parallel: &str) -> String {
+	format!(
+		"CREATE person:tobie SET name = 'Tobie', genre='m', company='SurrealDB';
+		CREATE person:jaime SET name = 'Jaime', genre='m', company='SurrealDB';
+		CREATE person:lizzie SET name = 'Lizzie', genre='f', company='SurrealDB';
+		DEFINE INDEX uniq_name ON TABLE person COLUMNS name UNIQUE;
+		DEFINE INDEX idx_genre ON TABLE person COLUMNS genre;
+		SELECT name FROM person {with} WHERE name = 'Jaime' OR genre = 'm' {parallel};
+	    SELECT name FROM person {with} WHERE name = 'Jaime' OR genre = 'm' {parallel} EXPLAIN FULL;
+		SELECT name FROM person {with} WHERE name = 'Jaime' AND genre = 'm' {parallel};
+	    SELECT name FROM person {with} WHERE name = 'Jaime' AND genre = 'm' {parallel} EXPLAIN FULL;"
+	)
+}
+
+fn three_multi_index_query(with: &str, parallel: &str) -> String {
+	format!("
 		CREATE person:tobie SET name = 'Tobie', genre='m', company='SurrealDB';
 		CREATE person:jaime SET name = 'Jaime', genre='m', company='SurrealDB';
 		CREATE person:lizzie SET name = 'Lizzie', genre='f', company='SurrealDB';
@@ -22,37 +160,47 @@ async fn test_select_where_iterate_multi_index(parallel: bool) -> Result<(), Err
 		DEFINE INDEX ft_company ON person FIELDS company SEARCH ANALYZER simple BM25;
 		DEFINE INDEX uniq_name ON TABLE person COLUMNS name UNIQUE;
 		DEFINE INDEX idx_genre ON TABLE person COLUMNS genre;
-		SELECT name FROM person WHERE name = 'Jaime' OR genre = 'm' OR company @@ 'surrealdb' {parallel};
+		SELECT name FROM person {with} WHERE name = 'Jaime' OR genre = 'm' OR company @@ 'surrealdb' {parallel};
-	    SELECT name FROM person WHERE name = 'Jaime' OR genre = 'm' OR company @@ 'surrealdb' {parallel} EXPLAIN FULL;"
+	    SELECT name FROM person {with} WHERE name = 'Jaime' OR genre = 'm' OR company @@ 'surrealdb' {parallel} EXPLAIN FULL;
-	);
+		SELECT name FROM person {with} WHERE name = 'Jaime' AND genre = 'm' AND company @@ 'surrealdb' {parallel};
-	let dbs = Datastore::new("memory").await?;
+	    SELECT name FROM person {with} WHERE name = 'Jaime' AND genre = 'm' AND company @@ 'surrealdb' {parallel} EXPLAIN FULL;")
-	let ses = Session::for_kv().with_ns("test").with_db("test");
+}
-	let res = &mut dbs.execute(&sql, &ses, None).await?;
+
-	assert_eq!(res.len(), 10);
+fn table_explain(fetch_count: usize) -> String {
-	//
+	format!(
-	for _ in 0..8 {
+		"[
-		let _ = res.remove(0).result?;
+			{{
+				detail: {{
+					table: 'person'
+				}},
+				operation: 'Iterate Table'
+			}},
+			{{
+				detail: {{
+					count: {fetch_count}
+				}},
+				operation: 'Fetch'
+			}}
+		]"
+	)
+}
+
+const THREE_TABLE_EXPLAIN: &str = "[
+	{
+		detail: {
+			table: 'person'
+		},
+		operation: 'Iterate Table'
+	},
+	{
+		detail: {
+			count: 3
+		},
+		operation: 'Fetch'
 	}
-	//
+]";
-	let tmp = res.remove(0).result?;
+
-	let val = Value::parse(
+const THREE_MULTI_INDEX_EXPLAIN: &str = "[
-		"[
-			{
-				name: 'Jaime'
-			},
-            {
-				name: 'Tobie'
-			},
-			{
-				name: 'Lizzie'
-			}
-		]",
-	);
-	assert_eq!(format!("{:#}", tmp), format!("{:#}", val));
-	//
-	let tmp = res.remove(0).result?;
-	let val = Value::parse(
-		"[
 				{
 					detail: {
 						plan: {
@@ -92,18 +240,95 @@ async fn test_select_where_iterate_multi_index(parallel: bool) -> Result<(), Err
 					},
 					operation: 'Fetch'
 				}
-			]",
+			]";
-	);
-	assert_eq!(format!("{:#}", tmp), format!("{:#}", val));
-	Ok(())
-}
 
-#[tokio::test]
+const SINGLE_INDEX_FT_EXPLAIN: &str = "[
-async fn select_where_iterate_multi_index() -> Result<(), Error> {
+				{
-	test_select_where_iterate_multi_index(false).await
+					detail: {
-}
+						plan: {
+							index: 'ft_company',
+							operator: '@@',
+							value: 'surrealdb'
+						},
+						table: 'person',
+					},
+					operation: 'Iterate Index'
+				},
+				{
+					detail: {
+						count: 1
+					},
+					operation: 'Fetch'
+				}
+			]";
 
-#[tokio::test]
+const SINGLE_INDEX_UNIQ_EXPLAIN: &str = "[
-async fn select_where_iterate_multi_index_parallel() -> Result<(), Error> {
+				{
-	test_select_where_iterate_multi_index(true).await
+					detail: {
-}
+						plan: {
+							index: 'uniq_name',
+							operator: '=',
+							value: 'Jaime'
+						},
+						table: 'person',
+					},
+					operation: 'Iterate Index'
+				},
+				{
+					detail: {
+						count: 1
+					},
+					operation: 'Fetch'
+				}
+			]";
+
+const SINGLE_INDEX_IDX_EXPLAIN: &str = "[
+	{
+		detail: {
+			plan: {
+				index: 'idx_genre',
+				operator: '=',
+				value: 'm'
+			},
+			table: 'person'
+		},
+		operation: 'Iterate Index'
+	},
+	{
+		detail: {
+			count: 1
+		},
+		operation: 'Fetch'
+	}
+]";
+
+const TWO_MULTI_INDEX_EXPLAIN: &str = "[
+				{
+					detail: {
+						plan: {
+							index: 'uniq_name',
+							operator: '=',
+							value: 'Jaime'
+						},
+						table: 'person',
+					},
+					operation: 'Iterate Index'
+				},
+                {
+					detail: {
+						plan: {
+							index: 'idx_genre',
+							operator: '=',
+							value: 'm'
+						},
+						table: 'person',
+					},
+					operation: 'Iterate Index'
+				},
+				{
+					detail: {
+						count: 2
+					},
+					operation: 'Fetch'
+				}
+			]";