From b8ff68b464b7fe3d46dbdf61d0f56d31ddfa8d1c Mon Sep 17 00:00:00 2001 From: Emmanuel Keller Date: Thu, 26 Oct 2023 22:34:28 +0100 Subject: [PATCH] Feat: Indexes used with the operators CONTAINS[ANY|ALL] (#2879) --- lib/src/idx/planner/executor.rs | 45 +++-- lib/src/idx/planner/iterators.rs | 90 +++++++-- lib/src/idx/planner/mod.rs | 3 + lib/src/idx/planner/plan.rs | 33 +++- lib/src/idx/planner/tree.rs | 136 +++++++++----- lib/tests/planner.rs | 304 ++++++++++++++++++++++++++++--- 6 files changed, 492 insertions(+), 119 deletions(-) diff --git a/lib/src/idx/planner/executor.rs b/lib/src/idx/planner/executor.rs index b9e5018a..ae7be53a 100644 --- a/lib/src/idx/planner/executor.rs +++ b/lib/src/idx/planner/executor.rs @@ -6,8 +6,8 @@ use crate::idx::ft::termdocs::TermsDocs; use crate::idx::ft::terms::TermId; use crate::idx::ft::{FtIndex, MatchRef}; use crate::idx::planner::iterators::{ - IndexEqualThingIterator, IndexRangeThingIterator, KnnThingIterator, MatchesThingIterator, - ThingIterator, UniqueEqualThingIterator, UniqueRangeThingIterator, + IndexEqualThingIterator, IndexRangeThingIterator, IndexUnionThingIterator, KnnThingIterator, + MatchesThingIterator, ThingIterator, UniqueEqualThingIterator, UniqueRangeThingIterator, }; use crate::idx::planner::plan::IndexOperator::Matches; use crate::idx::planner::plan::{IndexOperator, IndexOption, RangeValue}; @@ -199,8 +199,8 @@ impl QueryExecutor { IteratorEntry::Single(_, io) => { if let Some(ix) = self.index_definitions.get(&io.ir()) { match ix.index { - Index::Idx => Self::new_index_iterator(opt, ix, io.clone()), - Index::Uniq => Self::new_unique_index_iterator(opt, ix, io.clone()), + Index::Idx => Ok(Self::new_index_iterator(opt, ix, io.clone())), + Index::Uniq => Ok(Self::new_unique_index_iterator(opt, ix, io.clone())), Index::Search { .. } => self.new_search_index_iterator(ir, io.clone()).await, @@ -211,7 +211,7 @@ impl QueryExecutor { } } IteratorEntry::Range(_, ir, from, to) => { - Ok(self.new_range_iterator(opt, *ir, from, to)?) + Ok(self.new_range_iterator(opt, *ir, from, to)) } } } else { @@ -223,13 +223,15 @@ impl QueryExecutor { opt: &Options, ix: &DefineIndexStatement, io: IndexOption, - ) -> Result, Error> { + ) -> Option { match io.op() { - IndexOperator::Equality(array) => { - Ok(Some(ThingIterator::IndexEqual(IndexEqualThingIterator::new(opt, ix, array)?))) + IndexOperator::Equality(value) => { + Some(ThingIterator::IndexEqual(IndexEqualThingIterator::new(opt, ix, value))) } - IndexOperator::RangePart(_, _) => Ok(None), // TODO - _ => Ok(None), + IndexOperator::Union(value) => { + Some(ThingIterator::IndexUnion(IndexUnionThingIterator::new(opt, ix, value))) + } + _ => None, } } @@ -239,38 +241,35 @@ impl QueryExecutor { ir: IndexRef, from: &RangeValue, to: &RangeValue, - ) -> Result, Error> { + ) -> Option { if let Some(ix) = self.index_definitions.get(&ir) { match ix.index { Index::Idx => { - return Ok(Some(ThingIterator::IndexRange(IndexRangeThingIterator::new( + return Some(ThingIterator::IndexRange(IndexRangeThingIterator::new( opt, ix, from, to, - )))) + ))) } Index::Uniq => { - return Ok(Some(ThingIterator::UniqueRange(UniqueRangeThingIterator::new( + return Some(ThingIterator::UniqueRange(UniqueRangeThingIterator::new( opt, ix, from, to, - )))) + ))) } _ => {} } } - Ok(None) + None } fn new_unique_index_iterator( opt: &Options, ix: &DefineIndexStatement, io: IndexOption, - ) -> Result, Error> { + ) -> Option { match io.op() { - IndexOperator::Equality(array) => { - Ok(Some(ThingIterator::UniqueEqual(UniqueEqualThingIterator::new(opt, ix, array)?))) + IndexOperator::Equality(value) => { + Some(ThingIterator::UniqueEqual(UniqueEqualThingIterator::new(opt, ix, value))) } - IndexOperator::RangePart(_, _) => { - todo!() - } - _ => Ok(None), + _ => None, } } diff --git a/lib/src/idx/planner/iterators.rs b/lib/src/idx/planner/iterators.rs index 57aa7dd2..b8b2b7a7 100644 --- a/lib/src/idx/planner/iterators.rs +++ b/lib/src/idx/planner/iterators.rs @@ -16,6 +16,7 @@ use tokio::sync::RwLock; pub(crate) enum ThingIterator { IndexEqual(IndexEqualThingIterator), IndexRange(IndexRangeThingIterator), + IndexUnion(IndexUnionThingIterator), UniqueEqual(UniqueEqualThingIterator), UniqueRange(UniqueRangeThingIterator), Matches(MatchesThingIterator), @@ -33,6 +34,7 @@ impl ThingIterator { ThingIterator::UniqueEqual(i) => i.next_batch(tx).await, ThingIterator::IndexRange(i) => i.next_batch(tx, size).await, ThingIterator::UniqueRange(i) => i.next_batch(tx, size).await, + ThingIterator::IndexUnion(i) => i.next_batch(tx, size).await, ThingIterator::Matches(i) => i.next_batch(tx, size).await, ThingIterator::Knn(i) => i.next_batch(tx, size).await, } @@ -45,13 +47,32 @@ pub(crate) struct IndexEqualThingIterator { } impl IndexEqualThingIterator { - pub(super) fn new(opt: &Options, ix: &DefineIndexStatement, v: &Array) -> Result { - let beg = Index::prefix_ids_beg(opt.ns(), opt.db(), &ix.what, &ix.name, v); - let end = Index::prefix_ids_end(opt.ns(), opt.db(), &ix.what, &ix.name, v); - Ok(Self { + pub(super) fn new(opt: &Options, ix: &DefineIndexStatement, v: &Value) -> Self { + let a = Array::from(v.clone()); + let beg = Index::prefix_ids_beg(opt.ns(), opt.db(), &ix.what, &ix.name, &a); + let end = Index::prefix_ids_end(opt.ns(), opt.db(), &ix.what, &ix.name, &a); + Self { beg, end, - }) + } + } + + async fn next_scan( + txn: &Transaction, + beg: &mut Vec, + end: &[u8], + limit: u32, + ) -> Result, Error> { + let min = beg.clone(); + let max = end.to_owned(); + let res = txn.lock().await.scan(min..max, limit).await?; + if let Some((key, _)) = res.last() { + let mut key = key.clone(); + key.push(0x00); + *beg = key; + } + let res = res.iter().map(|(_, val)| (val.into(), NO_DOC_ID)).collect(); + Ok(res) } async fn next_batch( @@ -59,15 +80,7 @@ impl IndexEqualThingIterator { txn: &Transaction, limit: u32, ) -> Result, Error> { - let min = self.beg.clone(); - let max = self.end.clone(); - let res = txn.lock().await.scan(min..max, limit).await?; - if let Some((key, _)) = res.last() { - self.beg = key.clone(); - self.beg.push(0x00); - } - let res = res.iter().map(|(_, val)| (val.into(), NO_DOC_ID)).collect(); - Ok(res) + Self::next_scan(txn, &mut self.beg, &self.end, limit).await } } @@ -179,16 +192,57 @@ impl IndexRangeThingIterator { } } +pub(crate) struct IndexUnionThingIterator { + values: VecDeque<(Vec, Vec)>, + current: Option<(Vec, Vec)>, +} + +impl IndexUnionThingIterator { + pub(super) fn new(opt: &Options, ix: &DefineIndexStatement, a: &Array) -> Self { + // We create a VecDeque to hold the prefix keys (begin and end) for each value in the array. + let mut values: VecDeque<(Vec, Vec)> = + a.0.iter() + .map(|v| { + let a = Array::from(v.clone()); + let beg = Index::prefix_ids_beg(opt.ns(), opt.db(), &ix.what, &ix.name, &a); + let end = Index::prefix_ids_end(opt.ns(), opt.db(), &ix.what, &ix.name, &a); + (beg, end) + }) + .collect(); + let current = values.pop_front(); + Self { + values, + current, + } + } + + async fn next_batch( + &mut self, + txn: &Transaction, + limit: u32, + ) -> Result, Error> { + while let Some(r) = &mut self.current { + let res = IndexEqualThingIterator::next_scan(txn, &mut r.0, &r.1, limit).await?; + if !res.is_empty() { + return Ok(res); + } + self.current = self.values.pop_front(); + } + Ok(vec![]) + } +} + pub(crate) struct UniqueEqualThingIterator { key: Option, } impl UniqueEqualThingIterator { - pub(super) fn new(opt: &Options, ix: &DefineIndexStatement, a: &Array) -> Result { - let key = Index::new(opt.ns(), opt.db(), &ix.what, &ix.name, a, None).into(); - Ok(Self { + pub(super) fn new(opt: &Options, ix: &DefineIndexStatement, v: &Value) -> Self { + let a = Array::from(v.to_owned()); + let key = Index::new(opt.ns(), opt.db(), &ix.what, &ix.name, &a, None).into(); + Self { key: Some(key), - }) + } } async fn next_batch(&mut self, txn: &Transaction) -> Result, Error> { diff --git a/lib/src/idx/planner/mod.rs b/lib/src/idx/planner/mod.rs index fe197f84..c600aa1a 100644 --- a/lib/src/idx/planner/mod.rs +++ b/lib/src/idx/planner/mod.rs @@ -47,6 +47,9 @@ impl<'a> QueryPlanner<'a> { let mut exe = QueryExecutor::new(self.opt, txn, &t, im).await?; match PlanBuilder::build(node, self.with, with_indexes)? { Plan::SingleIndex(exp, io) => { + if io.require_distinct() { + self.requires_distinct = true; + } let ir = exe.add_iterator(IteratorEntry::Single(exp, io)); it.ingest(Iterable::Index(t.clone(), ir)); self.executors.insert(t.0.clone(), exe); diff --git a/lib/src/idx/planner/plan.rs b/lib/src/idx/planner/plan.rs index a1799671..fa16ab5c 100644 --- a/lib/src/idx/planner/plan.rs +++ b/lib/src/idx/planner/plan.rs @@ -146,7 +146,8 @@ pub(super) struct Inner { #[derive(Debug, Eq, PartialEq, Hash)] pub(super) enum IndexOperator { - Equality(Array), + Equality(Value), + Union(Array), RangePart(Operator, Value), Matches(String, Option), Knn(Array, u32), @@ -161,6 +162,10 @@ impl IndexOption { })) } + pub(super) fn require_distinct(&self) -> bool { + matches!(self.0.op, IndexOperator::Union(_)) + } + pub(super) fn ir(&self) -> IndexRef { self.0.ir } @@ -173,16 +178,24 @@ impl IndexOption { &self.0.id } + fn reduce_array(v: &Value) -> Value { + if let Value::Array(a) = v { + if a.len() == 1 { + return a[0].clone(); + } + } + v.clone() + } + pub(crate) fn explain(&self, e: &mut HashMap<&str, Value>) { match self.op() { - IndexOperator::Equality(a) => { - let v = if a.len() == 1 { - a[0].clone() - } else { - Value::Array(a.clone()) - }; + IndexOperator::Equality(v) => { e.insert("operator", Value::from(Operator::Equal.to_string())); - e.insert("value", v); + e.insert("value", Self::reduce_array(v)); + } + IndexOperator::Union(a) => { + e.insert("operator", Value::from("union")); + e.insert("value", Value::Array(a.clone())); } IndexOperator::Matches(qs, a) => { e.insert("operator", Value::from(Operator::Matches(*a).to_string())); @@ -303,13 +316,13 @@ mod tests { let io1 = IndexOption::new( 1, Idiom::from("a.b".to_string()), - IndexOperator::Equality(Array::from(vec!["test"])), + IndexOperator::Equality(Value::Array(Array::from(vec!["test"]))), ); let io2 = IndexOption::new( 1, Idiom::from("a.b".to_string()), - IndexOperator::Equality(Array::from(vec!["test"])), + IndexOperator::Equality(Value::Array(Array::from(vec!["test"]))), ); set.insert(io1); diff --git a/lib/src/idx/planner/tree.rs b/lib/src/idx/planner/tree.rs index eef85932..2915dd02 100644 --- a/lib/src/idx/planner/tree.rs +++ b/lib/src/idx/planner/tree.rs @@ -11,6 +11,28 @@ use std::sync::Arc; pub(super) struct Tree {} +#[derive(Clone, Copy)] +enum IdiomPosition { + Left, + Right, +} + +impl IdiomPosition { + // Reverses the operator for non commutative operators + fn transform(&self, op: &Operator) -> Operator { + match self { + IdiomPosition::Left => op.clone(), + IdiomPosition::Right => match op { + Operator::LessThan => Operator::MoreThan, + Operator::LessThanOrEqual => Operator::MoreThanOrEqual, + Operator::MoreThan => Operator::LessThan, + Operator::MoreThanOrEqual => Operator::LessThanOrEqual, + _ => op.clone(), + }, + } + } +} + impl Tree { /// Traverse all the conditions and extract every expression /// that can be resolved by an index. @@ -103,9 +125,9 @@ impl<'a> TreeBuilder<'a> { Value::Expression(e) => self.eval_expression(e).await, Value::Idiom(i) => self.eval_idiom(i).await, Value::Strand(_) | Value::Number(_) | Value::Bool(_) | Value::Thing(_) => { - Ok(Node::Scalar(v.to_owned())) + Ok(Node::Computed(v.to_owned())) } - Value::Array(a) => Ok(self.eval_array(a)), + Value::Array(a) => self.eval_array(a).await, Value::Subquery(s) => self.eval_subquery(s).await, Value::Param(p) => { let v = p.compute(self.ctx, self.opt, self.txn, None).await?; @@ -115,14 +137,12 @@ impl<'a> TreeBuilder<'a> { } } - fn eval_array(&mut self, a: &Array) -> Node { - // Check if it is a numeric vector + async fn eval_array(&mut self, a: &Array) -> Result { + let mut values = Vec::with_capacity(a.len()); for v in &a.0 { - if !v.is_number() { - return Node::Unsupported(format!("Unsupported array: {}", a)); - } + values.push(v.compute(self.ctx, self.opt, self.txn, None).await?); } - Node::Vector(a.to_owned()) + Ok(Node::Computed(Value::Array(Array::from(values)))) } async fn eval_idiom(&mut self, i: &Idiom) -> Result { @@ -163,9 +183,23 @@ impl<'a> TreeBuilder<'a> { } let mut io = None; if let Some((id, irs)) = left.is_indexed_field() { - io = self.lookup_index_option(irs.as_slice(), o, id, &right, e); + io = self.lookup_index_option( + irs.as_slice(), + o, + id, + &right, + e, + IdiomPosition::Left, + ); } else if let Some((id, irs)) = right.is_indexed_field() { - io = self.lookup_index_option(irs.as_slice(), o, id, &left, e); + io = self.lookup_index_option( + irs.as_slice(), + o, + id, + &left, + e, + IdiomPosition::Right, + ); }; Ok(Node::Expression { io, @@ -184,36 +218,17 @@ impl<'a> TreeBuilder<'a> { id: &Idiom, n: &Node, e: &Expression, + p: IdiomPosition, ) -> Option { for ir in irs { if let Some(ix) = self.index_map.definitions.get(ir) { let op = match &ix.index { - Index::Idx => Self::eval_index_operator(op, n), - Index::Uniq => Self::eval_index_operator(op, n), + Index::Idx => Self::eval_index_operator(op, n, p), + Index::Uniq => Self::eval_index_operator(op, n, p), Index::Search { .. - } => { - if let Some(v) = n.is_scalar() { - if let Operator::Matches(mr) = op { - Some(IndexOperator::Matches(v.clone().to_raw_string(), *mr)) - } else { - None - } - } else { - None - } - } - Index::MTree(_) => { - if let Operator::Knn(k) = op { - if let Node::Vector(a) = n { - Some(IndexOperator::Knn(a.clone(), *k)) - } else { - None - } - } else { - None - } - } + } => Self::eval_matches_operator(op, n), + Index::MTree(_) => Self::eval_knn_operator(op, n), }; if let Some(op) = op { let io = IndexOption::new(*ir, id.clone(), op); @@ -224,15 +239,45 @@ impl<'a> TreeBuilder<'a> { } None } + fn eval_matches_operator(op: &Operator, n: &Node) -> Option { + if let Some(v) = n.is_computed() { + if let Operator::Matches(mr) = op { + return Some(IndexOperator::Matches(v.clone().to_raw_string(), *mr)); + } + } + None + } - fn eval_index_operator(op: &Operator, n: &Node) -> Option { - if let Some(v) = n.is_scalar() { - match op { - Operator::Equal => Some(IndexOperator::Equality(Array::from(v.clone()))), - Operator::LessThan - | Operator::LessThanOrEqual - | Operator::MoreThan - | Operator::MoreThanOrEqual => Some(IndexOperator::RangePart(op.clone(), v.clone())), + fn eval_knn_operator(op: &Operator, n: &Node) -> Option { + if let Operator::Knn(k) = op { + if let Node::Computed(Value::Array(a)) = n { + return Some(IndexOperator::Knn(a.clone(), *k)); + } + } + None + } + + fn eval_index_operator(op: &Operator, n: &Node, p: IdiomPosition) -> Option { + if let Some(v) = n.is_computed() { + match (op, v, p) { + (Operator::Equal, v, _) => Some(IndexOperator::Equality(v.clone())), + (Operator::Contain, v, IdiomPosition::Left) => { + Some(IndexOperator::Equality(v.clone())) + } + (Operator::ContainAny, Value::Array(a), IdiomPosition::Left) => { + Some(IndexOperator::Union(a.clone())) + } + (Operator::ContainAll, Value::Array(a), IdiomPosition::Left) => { + Some(IndexOperator::Union(a.clone())) + } + ( + Operator::LessThan + | Operator::LessThanOrEqual + | Operator::MoreThan + | Operator::MoreThanOrEqual, + v, + p, + ) => Some(IndexOperator::RangePart(p.transform(op), v.clone())), _ => None, } } else { @@ -267,14 +312,13 @@ pub(super) enum Node { }, IndexedField(Idiom, Arc>), NonIndexedField, - Scalar(Value), - Vector(Array), + Computed(Value), Unsupported(String), } impl Node { - pub(super) fn is_scalar(&self) -> Option<&Value> { - if let Node::Scalar(v) = self { + pub(super) fn is_computed(&self) -> Option<&Value> { + if let Node::Computed(v) = self { Some(v) } else { None diff --git a/lib/tests/planner.rs b/lib/tests/planner.rs index 3f94b45e..97b2f670 100644 --- a/lib/tests/planner.rs +++ b/lib/tests/planner.rs @@ -5,11 +5,14 @@ mod helpers; use helpers::new_ds; use surrealdb::dbs::{Response, Session}; use surrealdb::err::Error; +use surrealdb::kvs::Datastore; use surrealdb::sql::Value; #[tokio::test] async fn select_where_iterate_three_multi_index() -> Result<(), Error> { - let mut res = execute_test(&three_multi_index_query("", ""), 12, 8).await?; + let dbs = new_ds().await?; + let mut res = execute_test(&dbs, &three_multi_index_query("", ""), 12).await?; + skip_ok(&mut res, 8)?; check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Tobie' }, { name: 'Lizzie' }]")?; // OR results check_result(&mut res, THREE_MULTI_INDEX_EXPLAIN)?; @@ -21,7 +24,9 @@ async fn select_where_iterate_three_multi_index() -> Result<(), Error> { #[tokio::test] async fn select_where_iterate_three_multi_index_parallel() -> Result<(), Error> { - let mut res = execute_test(&three_multi_index_query("", "PARALLEL"), 12, 8).await?; + let dbs = new_ds().await?; + let mut res = execute_test(&dbs, &three_multi_index_query("", "PARALLEL"), 12).await?; + skip_ok(&mut res, 8)?; // OR results check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Tobie' }, { name: 'Lizzie' }]")?; check_result(&mut res, THREE_MULTI_INDEX_EXPLAIN)?; @@ -33,12 +38,14 @@ async fn select_where_iterate_three_multi_index_parallel() -> Result<(), Error> #[tokio::test] async fn select_where_iterate_three_multi_index_with_all_index() -> Result<(), Error> { + let dbs = new_ds().await?; let mut res = execute_test( + &dbs, &three_multi_index_query("WITH INDEX uniq_name,idx_genre,ft_company", ""), 12, - 8, ) .await?; + skip_ok(&mut res, 8)?; // OR results check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Tobie' }, { name: 'Lizzie' }]")?; check_result(&mut res, THREE_MULTI_INDEX_EXPLAIN)?; @@ -50,8 +57,11 @@ async fn select_where_iterate_three_multi_index_with_all_index() -> Result<(), E #[tokio::test] async fn select_where_iterate_three_multi_index_with_one_ft_index() -> Result<(), Error> { + let dbs = new_ds().await?; let mut res = - execute_test(&three_multi_index_query("WITH INDEX ft_company", ""), 12, 8).await?; + execute_test(&dbs, &three_multi_index_query("WITH INDEX ft_company", ""), 12).await?; + skip_ok(&mut res, 8)?; + // OR results check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Lizzie' }, { name: 'Tobie' } ]")?; check_result(&mut res, THREE_TABLE_EXPLAIN)?; @@ -63,7 +73,11 @@ async fn select_where_iterate_three_multi_index_with_one_ft_index() -> Result<() #[tokio::test] async fn select_where_iterate_three_multi_index_with_one_index() -> Result<(), Error> { - let mut res = execute_test(&three_multi_index_query("WITH INDEX uniq_name", ""), 12, 8).await?; + let dbs = new_ds().await?; + let mut res = + execute_test(&dbs, &three_multi_index_query("WITH INDEX uniq_name", ""), 12).await?; + skip_ok(&mut res, 8)?; + // OR results check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Lizzie' }, { name: 'Tobie' } ]")?; check_result(&mut res, THREE_TABLE_EXPLAIN)?; @@ -75,7 +89,9 @@ async fn select_where_iterate_three_multi_index_with_one_index() -> Result<(), E #[tokio::test] async fn select_where_iterate_two_multi_index() -> Result<(), Error> { - let mut res = execute_test(&two_multi_index_query("", ""), 9, 5).await?; + let dbs = new_ds().await?; + let mut res = execute_test(&dbs, &two_multi_index_query("", ""), 9).await?; + skip_ok(&mut res, 5)?; // OR results check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Tobie' }]")?; check_result(&mut res, TWO_MULTI_INDEX_EXPLAIN)?; @@ -87,7 +103,9 @@ async fn select_where_iterate_two_multi_index() -> Result<(), Error> { #[tokio::test] async fn select_where_iterate_two_multi_index_with_one_index() -> Result<(), Error> { - let mut res = execute_test(&two_multi_index_query("WITH INDEX idx_genre", ""), 9, 5).await?; + let dbs = new_ds().await?; + let mut res = execute_test(&dbs, &two_multi_index_query("WITH INDEX idx_genre", ""), 9).await?; + skip_ok(&mut res, 5)?; // OR results check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Tobie' }]")?; check_result(&mut res, &table_explain(2))?; @@ -99,8 +117,10 @@ async fn select_where_iterate_two_multi_index_with_one_index() -> Result<(), Err #[tokio::test] async fn select_where_iterate_two_multi_index_with_two_index() -> Result<(), Error> { + let dbs = new_ds().await?; let mut res = - execute_test(&two_multi_index_query("WITH INDEX idx_genre,uniq_name", ""), 9, 5).await?; + execute_test(&dbs, &two_multi_index_query("WITH INDEX idx_genre,uniq_name", ""), 9).await?; + skip_ok(&mut res, 5)?; // OR results check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Tobie' }]")?; check_result(&mut res, TWO_MULTI_INDEX_EXPLAIN)?; @@ -112,7 +132,9 @@ async fn select_where_iterate_two_multi_index_with_two_index() -> Result<(), Err #[tokio::test] async fn select_where_iterate_two_no_index() -> Result<(), Error> { - let mut res = execute_test(&two_multi_index_query("WITH NOINDEX", ""), 9, 5).await?; + let dbs = new_ds().await?; + let mut res = execute_test(&dbs, &two_multi_index_query("WITH NOINDEX", ""), 9).await?; + skip_ok(&mut res, 5)?; // OR results check_result(&mut res, "[{ name: 'Jaime' }, { name: 'Tobie' }]")?; check_result(&mut res, &table_explain_no_index(2))?; @@ -123,19 +145,21 @@ async fn select_where_iterate_two_no_index() -> Result<(), Error> { } async fn execute_test( + dbs: &Datastore, sql: &str, expected_result: usize, - check_results: usize, ) -> Result, Error> { - let dbs = new_ds().await?; let ses = Session::owner().with_ns("test").with_db("test"); - let mut res = dbs.execute(sql, &ses, None).await?; + let res = dbs.execute(sql, &ses, None).await?; assert_eq!(res.len(), expected_result); - // Check that the setup is ok - for _ in 0..check_results { + Ok(res) +} + +fn skip_ok(res: &mut Vec, skip: usize) -> Result<(), Error> { + for _ in 0..skip { let _ = res.remove(0).result?; } - Ok(res) + Ok(()) } fn check_result(res: &mut Vec, expected: &str) -> Result<(), Error> { @@ -468,7 +492,9 @@ async fn select_range( explain: &str, result: &str, ) -> Result<(), Error> { - let mut res = execute_test(&range_test(unique, from_incl, to_incl), 8, 6).await?; + let dbs = new_ds().await?; + let mut res = execute_test(&dbs, &range_test(unique, from_incl, to_incl), 8).await?; + skip_ok(&mut res, 6)?; { let tmp = res.remove(0).result?; let val = Value::parse(explain); @@ -685,7 +711,9 @@ async fn select_single_range_operator( explain: &str, result: &str, ) -> Result<(), Error> { - let mut res = execute_test(&single_range_operator_test(unique, op), 6, 4).await?; + let dbs = new_ds().await?; + let mut res = execute_test(&dbs, &single_range_operator_test(unique, op), 6).await?; + skip_ok(&mut res, 4)?; { let tmp = res.remove(0).result?; let val = Value::parse(explain); @@ -862,11 +890,7 @@ async fn select_with_idiom_param_value() -> Result<(), Error> { ); let mut res = dbs.execute(&sql, &ses, None).await?; assert_eq!(res.len(), 6); - res.remove(0).result?; - res.remove(0).result?; - res.remove(0).result?; - res.remove(0).result?; - res.remove(0).result?; + skip_ok(&mut res, 5)?; let tmp = res.remove(0).result?; let val = Value::parse( r#"[ @@ -886,3 +910,239 @@ async fn select_with_idiom_param_value() -> Result<(), Error> { assert_eq!(format!("{:#}", tmp), format!("{:#}", val)); Ok(()) } + +const CONTAINS_CONTENT: &str = r#" + CREATE student:1 CONTENT { + marks: [ + { subject: "maths", mark: 50 }, + { subject: "english", mark: 40 }, + { subject: "tamil", mark: 45 } + ] + }; + CREATE student:2 CONTENT { + marks: [ + { subject: "maths", mark: 50 }, + { subject: "english", mark: 35 }, + { subject: "hindi", mark: 45 } + ] + }; + CREATE student:3 CONTENT { + marks: [ + { subject: "maths", mark: 50 }, + { subject: "hindi", mark: 30 }, + { subject: "tamil", mark: 45 } + ] + };"#; + +const CONTAINS_TABLE_EXPLAIN: &str = r"[ + { + detail: { + table: 'student' + }, + operation: 'Iterate Table' + }, + { + detail: { + reason: 'NO INDEX FOUND' + }, + operation: 'Fallback' + } + ]"; + +async fn test_contains( + dbs: &Datastore, + sql: &str, + index_explain: &str, + result: &str, +) -> Result<(), Error> { + let mut res = execute_test(&dbs, sql, 5).await?; + + { + let tmp = res.remove(0).result?; + let val = Value::parse(CONTAINS_TABLE_EXPLAIN); + assert_eq!(format!("{:#}", tmp), format!("{:#}", val)); + } + { + let tmp = res.remove(0).result?; + let val = Value::parse(result); + assert_eq!(format!("{:#}", tmp), format!("{:#}", val)); + } + skip_ok(&mut res, 1)?; + { + let tmp = res.remove(0).result?; + let val = Value::parse(index_explain); + assert_eq!(format!("{:#}", tmp), format!("{:#}", val)); + } + { + let tmp = res.remove(0).result?; + let val = Value::parse(result); + assert_eq!(format!("{:#}", tmp), format!("{:#}", val)); + } + Ok(()) +} + +#[tokio::test] +async fn select_contains() -> Result<(), Error> { + let dbs = new_ds().await?; + let mut res = execute_test(&dbs, CONTAINS_CONTENT, 3).await?; + skip_ok(&mut res, 3)?; + + const SQL: &str = r#" + SELECT id FROM student WHERE marks.*.subject CONTAINS "english" EXPLAIN; + SELECT id FROM student WHERE marks.*.subject CONTAINS "english"; + DEFINE INDEX subject_idx ON student COLUMNS marks.*.subject; + SELECT id FROM student WHERE marks.*.subject CONTAINS "english" EXPLAIN; + SELECT id FROM student WHERE marks.*.subject CONTAINS "english"; + "#; + + const INDEX_EXPLAIN: &str = r"[ + { + detail: { + table: 'student' + }, + detail: { + plan: { + index: 'subject_idx', + operator: '=', + value: 'english' + }, + table: 'student', + }, + operation: 'Iterate Index' + } + ]"; + const RESULT: &str = r"[ + { + id: student:1 + }, + { + id: student:2 + } + ]"; + + test_contains(&dbs, SQL, INDEX_EXPLAIN, RESULT).await +} + +#[tokio::test] +async fn select_contains_all() -> Result<(), Error> { + let dbs = new_ds().await?; + let mut res = execute_test(&dbs, CONTAINS_CONTENT, 3).await?; + skip_ok(&mut res, 3)?; + const SQL: &str = r#" + SELECT id FROM student WHERE marks.*.subject CONTAINSALL ["hindi", "maths"] EXPLAIN; + SELECT id FROM student WHERE marks.*.subject CONTAINSALL ["hindi", "maths"]; + DEFINE INDEX subject_idx ON student COLUMNS marks.*.subject; + SELECT id FROM student WHERE marks.*.subject CONTAINSALL ["hindi", "maths"] EXPLAIN; + SELECT id FROM student WHERE marks.*.subject CONTAINSALL ["hindi", "maths"]; + "#; + const INDEX_EXPLAIN: &str = r"[ + { + detail: { + table: 'student' + }, + detail: { + plan: { + index: 'subject_idx', + operator: 'union', + value: ['hindi', 'maths'] + }, + table: 'student', + }, + operation: 'Iterate Index' + } + ]"; + const RESULT: &str = r"[ + { + id: student:2 + }, + { + id: student:3 + } + ]"; + + test_contains(&dbs, SQL, INDEX_EXPLAIN, RESULT).await +} + +#[tokio::test] +async fn select_contains_any() -> Result<(), Error> { + let dbs = new_ds().await?; + let mut res = execute_test(&dbs, CONTAINS_CONTENT, 3).await?; + skip_ok(&mut res, 3)?; + const SQL: &str = r#" + SELECT id FROM student WHERE marks.*.subject CONTAINSANY ["tamil", "french"] EXPLAIN; + SELECT id FROM student WHERE marks.*.subject CONTAINSANY ["tamil", "french"]; + DEFINE INDEX subject_idx ON student COLUMNS marks.*.subject; + SELECT id FROM student WHERE marks.*.subject CONTAINSANY ["tamil", "french"] EXPLAIN; + SELECT id FROM student WHERE marks.*.subject CONTAINSANY ["tamil", "french"]; + "#; + const INDEX_EXPLAIN: &str = r"[ + { + detail: { + table: 'student' + }, + detail: { + plan: { + index: 'subject_idx', + operator: 'union', + value: ['tamil', 'french'] + }, + table: 'student', + }, + operation: 'Iterate Index' + } + ]"; + const RESULT: &str = r"[ + { + id: student:1 + }, + { + id: student:3 + } + ]"; + + test_contains(&dbs, SQL, INDEX_EXPLAIN, RESULT).await +} + +const CONTAINS_UNIQUE_CONTENT: &str = r#" + CREATE student:1 CONTENT { subject: "maths", mark: 50 }; + CREATE student:2 CONTENT { subject: "english", mark: 35 }; + CREATE student:3 CONTENT { subject: "hindi", mark: 30 };"#; + +#[tokio::test] +async fn select_unique_contains() -> Result<(), Error> { + let dbs = new_ds().await?; + let mut res = execute_test(&dbs, CONTAINS_UNIQUE_CONTENT, 3).await?; + skip_ok(&mut res, 3)?; + + const SQL: &str = r#" + SELECT id FROM student WHERE subject CONTAINS "english" EXPLAIN; + SELECT id FROM student WHERE subject CONTAINS "english"; + DEFINE INDEX subject_idx ON student COLUMNS subject UNIQUE; + SELECT id FROM student WHERE subject CONTAINS "english" EXPLAIN; + SELECT id FROM student WHERE subject CONTAINS "english"; + "#; + + const INDEX_EXPLAIN: &str = r"[ + { + detail: { + table: 'student' + }, + detail: { + plan: { + index: 'subject_idx', + operator: '=', + value: 'english' + }, + table: 'student', + }, + operation: 'Iterate Index' + } + ]"; + const RESULT: &str = r"[ + { + id: student:2 + } + ]"; + + test_contains(&dbs, SQL, INDEX_EXPLAIN, RESULT).await +}