noth/surrealpatch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Introduce query planner and indexing ()

Browse source
This commit is contained in:
Emmanuel Keller 2023-06-19 19:41:13 +01:00 committed by GitHub
parent 12cb551156
commit 668d3fd8fb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
125 changed files with 4864 additions and 2537 deletions

18
Cargo.lock generated
View file

@ -1387,6 +1387,12 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "deunicode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c1bba4f227a4a53d12b653f50ca7bf10c9119ae2aba56aff9e0338b5c98f36a"
[[package]]
name = "digest"
version = "0.10.7"
@ -3937,6 +3943,16 @@ dependencies = [
"smallvec",
]
[[package]]
name = "rust-stemmers"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
dependencies = [
"serde",
"serde_derive",
]
[[package]]
name = "rust_decimal"
version = "1.29.1"
@ -4531,6 +4547,7 @@ dependencies = [
"bung",
"chrono",
"criterion",
"deunicode",
"dmp",
"echodb",
"env_logger 0.10.0",
@ -4562,6 +4579,7 @@ dependencies = [
"roaring",
"rocksdb",
"rquickjs",
"rust-stemmers",
"rust_decimal",
"rustls 0.20.8",
"scrypt",

2
lib/Cargo.toml
View file

@ -63,6 +63,7 @@ bincode = "1.3.3"
channel = { version = "1.8.0", package = "async-channel" }
chrono = { version = "0.4.24", features = ["serde"] }
derive = { version = "0.8.0", package = "surrealdb-derive" }
deunicode = "1.3.3"
dmp = "0.2.0"
echodb = { version = "0.4.0", optional = true }
executor = { version = "1.5.1", package = "async-executor" }
@ -93,6 +94,7 @@ reqwest = { version = "0.11.18", default-features = false, features = ["json", "
roaring = { version = "0.10.1", features = ["serde"] }
rocksdb = { version = "0.21.0", optional = true }
rust_decimal = { version = "1.29.1", features = [ "maths" ] }
rust-stemmers = "1.2.0"
rustls = { version = "0.20.8", optional = true }
snap = "1.1.0"
scrypt = "0.11.0"

68
lib/src/ctx/context.rs
View file

@ -1,6 +1,10 @@
use crate::ctx::canceller::Canceller;
use crate::ctx::reason::Reason;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::idx::planner::executor::QueryExecutor;
use crate::sql::value::Value;
use crate::sql::Thing;
use std::borrow::Cow;
use std::collections::HashMap;
use std::fmt::{self, Debug};
@ -29,6 +33,14 @@ pub struct Context<'a> {
cancelled: Arc<AtomicBool>,
// A collection of read only values stored in this context.
values: HashMap<Cow<'static, str>, Cow<'a, Value>>,
// An optional transaction
transaction: Option<Transaction>,
// An optional query executor
query_executors: Option<Arc<HashMap<String, QueryExecutor>>>,
// An optional record id
thing: Option<&'a Thing>,
// An optional cursor document
cursor_doc: Option<&'a Value>,
}
impl<'a> Default for Context<'a> {
@ -44,6 +56,8 @@ impl<'a> Debug for Context<'a> {
.field("deadline", &self.deadline)
.field("cancelled", &self.cancelled)
.field("values", &self.values)
.field("thing", &self.thing)
.field("doc", &self.cursor_doc)
.finish()
}
}
@ -56,6 +70,10 @@ impl<'a> Context<'a> {
parent: None,
deadline: None,
cancelled: Arc::new(AtomicBool::new(false)),
transaction: None,
query_executors: None,
thing: None,
cursor_doc: None,
}
}
@ -66,6 +84,10 @@ impl<'a> Context<'a> {
parent: Some(parent),
deadline: parent.deadline,
cancelled: Arc::new(AtomicBool::new(false)),
transaction: parent.transaction.clone(),
query_executors: parent.query_executors.clone(),
thing: parent.thing,
cursor_doc: parent.cursor_doc,
}
}
@ -91,6 +113,29 @@ impl<'a> Context<'a> {
self.add_deadline(Instant::now() + timeout)
}
pub fn add_transaction(&mut self, txn: Option<&Transaction>) {
if let Some(txn) = txn {
self.transaction = Some(txn.clone());
}
}
pub fn add_thing(&mut self, thing: &'a Thing) {
self.thing = Some(thing);
}
/// Add a cursor document to this context.
/// Usage: A new child context is created by an iterator for each document.
/// The iterator sets the value of the current document (known as cursor document).
/// The cursor document is copied do the child contexts.
pub(crate) fn add_cursor_doc(&mut self, doc: &'a Value) {
self.cursor_doc = Some(doc);
}
/// Set the query executors
pub(crate) fn set_query_executors(&mut self, executors: HashMap<String, QueryExecutor>) {
self.query_executors = Some(Arc::new(executors));
}
/// Add a value to the context. It overwrites any previously set values
/// with the same key.
pub fn add_value<K, V>(&mut self, key: K, value: V)
@ -107,6 +152,29 @@ impl<'a> Context<'a> {
self.deadline.map(|v| v.saturating_duration_since(Instant::now()))
}
pub fn clone_transaction(&self) -> Result<Transaction, Error> {
match &self.transaction {
None => Err(Error::NoTx),
Some(txn) => Ok(txn.clone()),
}
}
pub fn thing(&self) -> Option<&Thing> {
self.thing
}
pub fn doc(&self) -> Option<&Value> {
self.cursor_doc
}
pub(crate) fn get_query_executor(&self, tb: &str) -> Option<&QueryExecutor> {
if let Some(qe) = &self.query_executors {
qe.get(tb)
} else {
None
}
}
/// Check if the context is done. If it returns `None` the operation may
/// proceed, otherwise the operation should be stopped.
pub fn done(&self) -> Option<Reason> {

7
lib/src/dbs/channel.rs
View file

@ -3,7 +3,6 @@ use crate::dbs::Iterable;
use crate::dbs::Operable;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::key::graph;
use crate::key::thing;
@ -19,11 +18,12 @@ impl Iterable {
self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_stm: &Statement<'_>,
chn: Sender<(Option<Thing>, Operable)>,
) -> Result<(), Error> {
if ctx.is_ok() {
// Clone transaction
let txn = ctx.clone_transaction()?;
match self {
Iterable::Value(v) => {
// Pass the value through
@ -351,6 +351,9 @@ impl Iterable {
}
}
}
Iterable::Index(_t, _p) => {
todo!()
}
}
}
Ok(())

17
lib/src/dbs/executor.rs
View file

@ -33,10 +33,6 @@ impl<'a> Executor<'a> {
}
}
fn txn(&self) -> Transaction {
self.txn.clone().expect("unreachable: txn was None after successful begin")
}
/// # Return
/// - true if a new transaction has begun
/// - false if
@ -247,7 +243,10 @@ impl<'a> Executor<'a> {
// Check if the variable is a protected variable
let res = match PROTECTED_PARAM_NAMES.contains(&stm.name.as_str()) {
// The variable isn't protected and can be stored
false => stm.compute(&ctx, &opt, &self.txn(), None).await,
false => {
ctx.add_transaction(self.txn.as_ref());
stm.compute(&ctx, &opt).await
}
// The user tried to set a protected variable
true => Err(Error::InvalidParam {
// Move the parameter name, as we no longer need it
@ -305,8 +304,9 @@ impl<'a> Executor<'a> {
// Set statement timeout
let mut ctx = Context::new(&ctx);
ctx.add_timeout(timeout);
ctx.add_transaction(self.txn.as_ref());
// Process the statement
let res = stm.compute(&ctx, &opt, &self.txn(), None).await;
let res = stm.compute(&ctx, &opt).await;
// Catch statement timeout
match ctx.is_timedout() {
true => Err(Error::QueryTimedout),
@ -314,7 +314,10 @@ impl<'a> Executor<'a> {
}
}
// There is no timeout clause
None => stm.compute(&ctx, &opt, &self.txn(), None).await,
None => {
ctx.add_transaction(self.txn.as_ref());
stm.compute(&ctx, &opt).await
}
};
// Catch global timeout
let res = match ctx.is_timedout() {

772
lib/src/dbs/iterate.rs
View file

@ -4,13 +4,14 @@ use crate::dbs::Iterator;
use crate::dbs::Operable;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::idx::planner::plan::Plan;
use crate::key::graph;
use crate::key::thing;
use crate::sql::dir::Dir;
use crate::sql::thing::Thing;
use crate::sql::value::Value;
use crate::sql::{Edges, Range, Table};
use std::ops::Bound;
impl Iterable {
@ -18,332 +19,473 @@ impl Iterable {
self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
ite: &mut Iterator,
) -> Result<(), Error> {
if ctx.is_ok() {
match self {
Iterable::Value(v) => {
// Pass the value through
let val = Operable::Value(v);
// Process the document record
ite.process(ctx, opt, txn, stm, None, val).await;
}
Iterable::Thing(v) => {
// Check that the table exists
txn.lock().await.check_ns_db_tb(opt.ns(), opt.db(), &v.tb, opt.strict).await?;
// Fetch the data from the store
let key = thing::new(opt.ns(), opt.db(), &v.tb, &v.id);
let val = txn.clone().lock().await.get(key).await?;
// Parse the data from the store
let val = Operable::Value(match val {
Some(v) => Value::from(v),
None => Value::None,
});
// Process the document record
ite.process(ctx, opt, txn, stm, Some(v), val).await;
}
Iterable::Value(v) => Self::iterate_value(ctx, opt, stm, v, ite).await,
Iterable::Thing(v) => Self::iterate_thing(ctx, opt, stm, v, ite).await?,
Iterable::Mergeable(v, o) => {
// Check that the table exists
txn.lock().await.check_ns_db_tb(opt.ns(), opt.db(), &v.tb, opt.strict).await?;
// Fetch the data from the store
let key = thing::new(opt.ns(), opt.db(), &v.tb, &v.id);
let val = txn.clone().lock().await.get(key).await?;
// Parse the data from the store
let x = match val {
Some(v) => Value::from(v),
None => Value::None,
};
// Create a new operable value
let val = Operable::Mergeable(x, o);
// Process the document record
ite.process(ctx, opt, txn, stm, Some(v), val).await;
Self::iterate_mergeable(ctx, opt, stm, v, o, ite).await?;
}
Iterable::Relatable(f, v, w) => {
// Check that the table exists
txn.lock().await.check_ns_db_tb(opt.ns(), opt.db(), &v.tb, opt.strict).await?;
// Fetch the data from the store
let key = thing::new(opt.ns(), opt.db(), &v.tb, &v.id);
let val = txn.clone().lock().await.get(key).await?;
// Parse the data from the store
let x = match val {
Some(v) => Value::from(v),
None => Value::None,
};
// Create a new operable value
let val = Operable::Relatable(f, x, w);
// Process the document record
ite.process(ctx, opt, txn, stm, Some(v), val).await;
Self::iterate_relatable(ctx, opt, stm, f, v, w, ite).await?
}
Iterable::Table(v) => {
// Check that the table exists
txn.lock().await.check_ns_db_tb(opt.ns(), opt.db(), &v, opt.strict).await?;
// Prepare the start and end keys
let beg = thing::prefix(opt.ns(), opt.db(), &v);
let end = thing::suffix(opt.ns(), opt.db(), &v);
// Prepare the next holder key
let mut nxt: Option<Vec<u8>> = None;
// Loop until no more keys
loop {
// Check if the context is finished
if ctx.is_done() {
break;
}
// Get the next 1000 key-value entries
let res = match nxt {
None => {
let min = beg.clone();
let max = end.clone();
txn.clone().lock().await.scan(min..max, 1000).await?
}
Some(ref mut beg) => {
beg.push(0x00);
let min = beg.clone();
let max = end.clone();
txn.clone().lock().await.scan(min..max, 1000).await?
}
};
// If there are key-value entries then fetch them
if !res.is_empty() {
// Get total results
let n = res.len();
// Loop over results
for (i, (k, v)) in res.into_iter().enumerate() {
// Check the context
if ctx.is_done() {
break;
}
// Ready the next
if n == i + 1 {
nxt = Some(k.clone());
}
// Parse the data from the store
let key: crate::key::thing::Thing = (&k).into();
let val: crate::sql::value::Value = (&v).into();
let rid = Thing::from((key.tb, key.id));
// Create a new operable value
let val = Operable::Value(val);
// Process the record
ite.process(ctx, opt, txn, stm, Some(rid), val).await;
}
continue;
}
break;
}
}
Iterable::Range(v) => {
// Check that the table exists
txn.lock().await.check_ns_db_tb(opt.ns(), opt.db(), &v.tb, opt.strict).await?;
// Prepare the range start key
let beg = match &v.beg {
Bound::Unbounded => thing::prefix(opt.ns(), opt.db(), &v.tb),
Bound::Included(id) => {
thing::new(opt.ns(), opt.db(), &v.tb, id).encode().unwrap()
}
Bound::Excluded(id) => {
let mut key =
thing::new(opt.ns(), opt.db(), &v.tb, id).encode().unwrap();
key.push(0x00);
key
}
};
// Prepare the range end key
let end = match &v.end {
Bound::Unbounded => thing::suffix(opt.ns(), opt.db(), &v.tb),
Bound::Excluded(id) => {
thing::new(opt.ns(), opt.db(), &v.tb, id).encode().unwrap()
}
Bound::Included(id) => {
let mut key =
thing::new(opt.ns(), opt.db(), &v.tb, id).encode().unwrap();
key.push(0x00);
key
}
};
// Prepare the next holder key
let mut nxt: Option<Vec<u8>> = None;
// Loop until no more keys
loop {
// Check if the context is finished
if ctx.is_done() {
break;
}
// Get the next 1000 key-value entries
let res = match nxt {
None => {
let min = beg.clone();
let max = end.clone();
txn.clone().lock().await.scan(min..max, 1000).await?
}
Some(ref mut beg) => {
beg.push(0x00);
let min = beg.clone();
let max = end.clone();
txn.clone().lock().await.scan(min..max, 1000).await?
}
};
// If there are key-value entries then fetch them
if !res.is_empty() {
// Get total results
let n = res.len();
// Loop over results
for (i, (k, v)) in res.into_iter().enumerate() {
// Check the context
if ctx.is_done() {
break;
}
// Ready the next
if n == i + 1 {
nxt = Some(k.clone());
}
// Parse the data from the store
let key: crate::key::thing::Thing = (&k).into();
let val: crate::sql::value::Value = (&v).into();
let rid = Thing::from((key.tb, key.id));
// Create a new operable value
let val = Operable::Value(val);
// Process the record
ite.process(ctx, opt, txn, stm, Some(rid), val).await;
}
continue;
}
break;
}
}
Iterable::Edges(e) => {
// Pull out options
let ns = opt.ns();
let db = opt.db();
let tb = &e.from.tb;
let id = &e.from.id;
// Fetch start and end key pairs
let keys = match e.what.len() {
0 => match e.dir {
// /ns/db/tb/id
Dir::Both => {
vec![(graph::prefix(ns, db, tb, id), graph::suffix(ns, db, tb, id))]
}
// /ns/db/tb/id/IN
Dir::In => vec![(
graph::egprefix(ns, db, tb, id, &e.dir),
graph::egsuffix(ns, db, tb, id, &e.dir),
)],
// /ns/db/tb/id/OUT
Dir::Out => vec![(
graph::egprefix(ns, db, tb, id, &e.dir),
graph::egsuffix(ns, db, tb, id, &e.dir),
)],
},
_ => match e.dir {
// /ns/db/tb/id/IN/TB
Dir::In => e
.what
.iter()
.map(|v| v.to_string())
.map(|v| {
(
graph::ftprefix(ns, db, tb, id, &e.dir, &v),
graph::ftsuffix(ns, db, tb, id, &e.dir, &v),
)
})
.collect::<Vec<_>>(),
// /ns/db/tb/id/OUT/TB
Dir::Out => e
.what
.iter()
.map(|v| v.to_string())
.map(|v| {
(
graph::ftprefix(ns, db, tb, id, &e.dir, &v),
graph::ftsuffix(ns, db, tb, id, &e.dir, &v),
)
})
.collect::<Vec<_>>(),
// /ns/db/tb/id/IN/TB, /ns/db/tb/id/OUT/TB
Dir::Both => e
.what
.iter()
.map(|v| v.to_string())
.flat_map(|v| {
vec![
(
graph::ftprefix(ns, db, tb, id, &Dir::In, &v),
graph::ftsuffix(ns, db, tb, id, &Dir::In, &v),
),
(
graph::ftprefix(ns, db, tb, id, &Dir::Out, &v),
graph::ftsuffix(ns, db, tb, id, &Dir::Out, &v),
),
]
})
.collect::<Vec<_>>(),
},
};
//
for (beg, end) in keys.iter() {
// Prepare the next holder key
let mut nxt: Option<Vec<u8>> = None;
// Loop until no more keys
loop {
// Check if the context is finished
if ctx.is_done() {
break;
}
// Get the next 1000 key-value entries
let res = match nxt {
None => {
let min = beg.clone();
let max = end.clone();
txn.clone().lock().await.scan(min..max, 1000).await?
}
Some(ref mut beg) => {
beg.push(0x00);
let min = beg.clone();
let max = end.clone();
txn.clone().lock().await.scan(min..max, 1000).await?
}
};
// If there are key-value entries then fetch them
if !res.is_empty() {
// Get total results
let n = res.len();
// Exit when settled
if n == 0 {
break;
}
// Loop over results
for (i, (k, _)) in res.into_iter().enumerate() {
// Check the context
if ctx.is_done() {
break;
}
// Ready the next
if n == i + 1 {
nxt = Some(k.clone());
}
// Parse the data from the store
let gra: crate::key::graph::Graph = (&k).into();
// Fetch the data from the store
let key = thing::new(opt.ns(), opt.db(), gra.ft, &gra.fk);
let val = txn.clone().lock().await.get(key).await?;
let rid = Thing::from((gra.ft, gra.fk));
// Parse the data from the store
let val = Operable::Value(match val {
Some(v) => Value::from(v),
None => Value::None,
});
// Process the record
ite.process(ctx, opt, txn, stm, Some(rid), val).await;
}
continue;
}
break;
}
}
Iterable::Table(v) => Self::iterate_table(ctx, opt, stm, v, ite).await?,
Iterable::Range(v) => Self::iterate_range(ctx, opt, stm, v, ite).await?,
Iterable::Edges(e) => Self::iterate_edge(ctx, opt, stm, e, ite).await?,
Iterable::Index(t, p) => {
Self::iterate_index(ctx, opt, stm, t, p, ite).await?;
}
}
}
Ok(())
}
async fn iterate_value(
ctx: &Context<'_>,
opt: &Options,
stm: &Statement<'_>,
v: Value,
ite: &mut Iterator,
) {
// Pass the value through
let val = Operable::Value(v);
// Process the document record
ite.process(ctx, opt, stm, val).await;
}
async fn iterate_thing(
ctx: &Context<'_>,
opt: &Options,
stm: &Statement<'_>,
v: Thing,
ite: &mut Iterator,
) -> Result<(), Error> {
// Clone transaction
let txn = ctx.clone_transaction()?;
// Check that the table exists
txn.lock().await.check_ns_db_tb(opt.ns(), opt.db(), &v.tb, opt.strict).await?;
// Fetch the data from the store
let key = thing::new(opt.ns(), opt.db(), &v.tb, &v.id);
let val = txn.clone().lock().await.get(key).await?;
// Parse the data from the store
let val = Operable::Value(match val {
Some(v) => Value::from(v),
None => Value::None,
});
// Get the optional query executor
let mut child_ctx = Context::new(ctx);
child_ctx.add_thing(&v);
// Process the document record
ite.process(&child_ctx, opt, stm, val).await;
Ok(())
}
async fn iterate_mergeable(
ctx: &Context<'_>,
opt: &Options,
stm: &Statement<'_>,
v: Thing,
o: Value,
ite: &mut Iterator,
) -> Result<(), Error> {
// Clone transaction
let txn = ctx.clone_transaction()?;
// Check that the table exists
txn.lock().await.check_ns_db_tb(opt.ns(), opt.db(), &v.tb, opt.strict).await?;
// Fetch the data from the store
let key = thing::new(opt.ns(), opt.db(), &v.tb, &v.id);
let val = txn.clone().lock().await.get(key).await?;
// Parse the data from the store
let x = match val {
Some(v) => Value::from(v),
None => Value::None,
};
// Create a new operable value
let val = Operable::Mergeable(x, o);
// Create a new context to process the operable
let mut child_ctx = Context::new(ctx);
child_ctx.add_thing(&v);
// Process the document record
ite.process(&child_ctx, opt, stm, val).await;
Ok(())
}
async fn iterate_relatable(
ctx: &Context<'_>,
opt: &Options,
stm: &Statement<'_>,
f: Thing,
v: Thing,
w: Thing,
ite: &mut Iterator,
) -> Result<(), Error> {
// Clone transaction
let txn = ctx.clone_transaction()?;
// Check that the table exists
txn.lock().await.check_ns_db_tb(opt.ns(), opt.db(), &v.tb, opt.strict).await?;
// Fetch the data from the store
let key = thing::new(opt.ns(), opt.db(), &v.tb, &v.id);
let val = txn.clone().lock().await.get(key).await?;
// Parse the data from the store
let x = match val {
Some(v) => Value::from(v),
None => Value::None,
};
// Create a new operable value
let val = Operable::Relatable(f, x, w);
// Create the child context
let mut child_ctx = Context::new(ctx);
child_ctx.add_thing(&v);
// Process the document record
ite.process(&child_ctx, opt, stm, val).await;
Ok(())
}
async fn iterate_table(
ctx: &Context<'_>,
opt: &Options,
stm: &Statement<'_>,
v: Table,
ite: &mut Iterator,
) -> Result<(), Error> {
// Clone transaction
let txn = ctx.clone_transaction()?;
// Check that the table exists
txn.lock().await.check_ns_db_tb(opt.ns(), opt.db(), &v, opt.strict).await?;
// Prepare the start and end keys
let beg = thing::prefix(opt.ns(), opt.db(), &v);
let end = thing::suffix(opt.ns(), opt.db(), &v);
// Prepare the next holder key
let mut nxt: Option<Vec<u8>> = None;
// Loop until no more keys
loop {
// Check if the context is finished
if ctx.is_done() {
break;
}
// Get the next 1000 key-value entries
let res = match nxt {
None => {
let min = beg.clone();
let max = end.clone();
txn.clone().lock().await.scan(min..max, 1000).await?
}
Some(ref mut beg) => {
beg.push(0x00);
let min = beg.clone();
let max = end.clone();
txn.clone().lock().await.scan(min..max, 1000).await?
}
};
// If there are key-value entries then fetch them
if !res.is_empty() {
// Get total results
let n = res.len();
// Loop over results
for (i, (k, v)) in res.into_iter().enumerate() {
// Check the context
if ctx.is_done() {
break;
}
// Ready the next
if n == i + 1 {
nxt = Some(k.clone());
}
// Parse the data from the store
let key: crate::key::thing::Thing = (&k).into();
let val: crate::sql::value::Value = (&v).into();
let rid = Thing::from((key.tb, key.id));
// Create a new operable value
let val = Operable::Value(val);
let mut child_ctx = Context::new(&ctx);
child_ctx.add_thing(&rid);
// Process the record
ite.process(&child_ctx, opt, stm, val).await;
}
continue;
}
break;
}
Ok(())
}
async fn iterate_range(
ctx: &Context<'_>,
opt: &Options,
stm: &Statement<'_>,
v: Range,
ite: &mut Iterator,
) -> Result<(), Error> {
// Clone transaction
let txn = ctx.clone_transaction()?;
// Check that the table exists
txn.lock().await.check_ns_db_tb(opt.ns(), opt.db(), &v.tb, opt.strict).await?;
// Prepare the range start key
let beg = match &v.beg {
Bound::Unbounded => thing::prefix(opt.ns(), opt.db(), &v.tb),
Bound::Included(id) => thing::new(opt.ns(), opt.db(), &v.tb, id).encode().unwrap(),
Bound::Excluded(id) => {
let mut key = thing::new(opt.ns(), opt.db(), &v.tb, id).encode().unwrap();
key.push(0x00);
key
}
};
// Prepare the range end key
let end = match &v.end {
Bound::Unbounded => thing::suffix(opt.ns(), opt.db(), &v.tb),
Bound::Excluded(id) => thing::new(opt.ns(), opt.db(), &v.tb, id).encode().unwrap(),
Bound::Included(id) => {
let mut key = thing::new(opt.ns(), opt.db(), &v.tb, id).encode().unwrap();
key.push(0x00);
key
}
};
// Prepare the next holder key
let mut nxt: Option<Vec<u8>> = None;
// Loop until no more keys
loop {
// Check if the context is finished
if ctx.is_done() {
break;
}
// Get the next 1000 key-value entries
let res = match nxt {
None => {
let min = beg.clone();
let max = end.clone();
txn.clone().lock().await.scan(min..max, 1000).await?
}
Some(ref mut beg) => {
beg.push(0x00);
let min = beg.clone();
let max = end.clone();
txn.clone().lock().await.scan(min..max, 1000).await?
}
};
// If there are key-value entries then fetch them
if !res.is_empty() {
// Get total results
let n = res.len();
// Loop over results
for (i, (k, v)) in res.into_iter().enumerate() {
// Check the context
if ctx.is_done() {
break;
}
// Ready the next
if n == i + 1 {
nxt = Some(k.clone());
}
// Parse the data from the store
let key: crate::key::thing::Thing = (&k).into();
let val: crate::sql::value::Value = (&v).into();
let rid = Thing::from((key.tb, key.id));
let mut ctx = Context::new(ctx);
ctx.add_thing(&rid);
// Create a new operable value
let val = Operable::Value(val);
// Process the record
ite.process(&ctx, opt, stm, val).await;
}
continue;
}
break;
}
Ok(())
}
async fn iterate_edge(
ctx: &Context<'_>,
opt: &Options,
stm: &Statement<'_>,
e: Edges,
ite: &mut Iterator,
) -> Result<(), Error> {
// Pull out options
let ns = opt.ns();
let db = opt.db();
let tb = &e.from.tb;
let id = &e.from.id;
// Fetch start and end key pairs
let keys = match e.what.len() {
0 => match e.dir {
// /ns/db/tb/id
Dir::Both => {
vec![(graph::prefix(ns, db, tb, id), graph::suffix(ns, db, tb, id))]
}
// /ns/db/tb/id/IN
Dir::In => vec![(
graph::egprefix(ns, db, tb, id, &e.dir),
graph::egsuffix(ns, db, tb, id, &e.dir),
)],
// /ns/db/tb/id/OUT
Dir::Out => vec![(
graph::egprefix(ns, db, tb, id, &e.dir),
graph::egsuffix(ns, db, tb, id, &e.dir),
)],
},
_ => match e.dir {
// /ns/db/tb/id/IN/TB
Dir::In => e
.what
.iter()
.map(|v| v.to_string())
.map(|v| {
(
graph::ftprefix(ns, db, tb, id, &e.dir, &v),
graph::ftsuffix(ns, db, tb, id, &e.dir, &v),
)
})
.collect::<Vec<_>>(),
// /ns/db/tb/id/OUT/TB
Dir::Out => e
.what
.iter()
.map(|v| v.to_string())
.map(|v| {
(
graph::ftprefix(ns, db, tb, id, &e.dir, &v),
graph::ftsuffix(ns, db, tb, id, &e.dir, &v),
)
})
.collect::<Vec<_>>(),
// /ns/db/tb/id/IN/TB, /ns/db/tb/id/OUT/TB
Dir::Both => e
.what
.iter()
.map(|v| v.to_string())
.flat_map(|v| {
vec![
(
graph::ftprefix(ns, db, tb, id, &Dir::In, &v),
graph::ftsuffix(ns, db, tb, id, &Dir::In, &v),
),
(
graph::ftprefix(ns, db, tb, id, &Dir::Out, &v),
graph::ftsuffix(ns, db, tb, id, &Dir::Out, &v),
),
]
})
.collect::<Vec<_>>(),
},
};
//
for (beg, end) in keys.iter() {
// Prepare the next holder key
let mut nxt: Option<Vec<u8>> = None;
// Loop until no more keys
loop {
// Check if the context is finished
if ctx.is_done() {
break;
}
// Get the next 1000 key-value entries
let res = match nxt {
None => {
let min = beg.clone();
let max = end.clone();
ctx.clone_transaction()?.lock().await.scan(min..max, 1000).await?
}
Some(ref mut beg) => {
beg.push(0x00);
let min = beg.clone();
let max = end.clone();
ctx.clone_transaction()?.lock().await.scan(min..max, 1000).await?
}
};
// If there are key-value entries then fetch them
if !res.is_empty() {
// Get total results
let n = res.len();
// Exit when settled
if n == 0 {
break;
}
// Loop over results
for (i, (k, _)) in res.into_iter().enumerate() {
// Check the context
if ctx.is_done() {
break;
}
// Ready the next
if n == i + 1 {
nxt = Some(k.clone());
}
// Parse the data from the store
let gra: crate::key::graph::Graph = (&k).into();
// Fetch the data from the store
let key = thing::new(opt.ns(), opt.db(), gra.ft, &gra.fk);
let val = ctx.clone_transaction()?.lock().await.get(key).await?;
let rid = Thing::from((gra.ft, gra.fk));
let mut ctx = Context::new(ctx);
ctx.add_thing(&rid);
// Parse the data from the store
let val = Operable::Value(match val {
Some(v) => Value::from(v),
None => Value::None,
});
// Process the record
ite.process(&ctx, opt, stm, val).await;
}
continue;
}
break;
}
}
Ok(())
}
async fn iterate_index(
ctx: &Context<'_>,
opt: &Options,
stm: &Statement<'_>,
table: Table,
plan: Plan,
ite: &mut Iterator,
) -> Result<(), Error> {
let txn = ctx.clone_transaction()?;
// Check that the table exists
txn.lock().await.check_ns_db_tb(opt.ns(), opt.db(), &table.0, opt.strict).await?;
let mut iterator = plan.new_iterator(opt, &txn).await?;
let mut things = iterator.next_batch(&txn, 1000).await?;
while !things.is_empty() {
// Check if the context is finished
if ctx.is_done() {
break;
}
for thing in things {
// Check the context
if ctx.is_done() {
break;
}
// If the record is from another table we can skip
if !thing.tb.eq(table.as_str()) {
continue;
}
// Fetch the data from the store
let key = thing::new(opt.ns(), opt.db(), &table.0, &thing.id);
let val = txn.lock().await.get(key.clone()).await?;
let rid = Thing::from((key.tb, key.id));
let mut ctx = Context::new(&ctx);
ctx.add_thing(&rid);
// Parse the data from the store
let val = Operable::Value(match val {
Some(v) => Value::from(v),
None => Value::None,
});
// Process the document record
ite.process(&ctx, opt, stm, val).await;
}
// Collect the next batch of ids
things = iterator.next_batch(&txn, 1000).await?;
}
Ok(())
}
}

152
lib/src/dbs/iterator.rs
View file

@ -2,10 +2,10 @@ use crate::ctx::Canceller;
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::dbs::LOG;
use crate::doc::Document;
use crate::err::Error;
use crate::idx::planner::plan::Plan;
use crate::sql::array::Array;
use crate::sql::edges::Edges;
use crate::sql::field::Field;
@ -13,10 +13,11 @@ use crate::sql::range::Range;
use crate::sql::table::Table;
use crate::sql::thing::Thing;
use crate::sql::value::Value;
use crate::sql::Object;
use async_recursion::async_recursion;
use std::borrow::Cow;
use std::cmp::Ordering;
use std::collections::BTreeMap;
use std::collections::{BTreeMap, HashMap};
use std::mem;
pub(crate) enum Iterable {
@ -27,6 +28,7 @@ pub(crate) enum Iterable {
Edges(Edges),
Mergeable(Thing, Value),
Relatable(Thing, Thing, Thing),
Index(Table, Plan),
}
pub(crate) enum Operable {
@ -73,7 +75,6 @@ impl Iterator {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<Value, Error> {
// Log the statement
@ -82,27 +83,33 @@ impl Iterator {
let mut run = Context::new(ctx);
self.run = run.add_cancel();
// Process the query LIMIT clause
self.setup_limit(&run, opt, txn, stm).await?;
self.setup_limit(ctx, opt, stm).await?;
// Process the query START clause
self.setup_start(&run, opt, txn, stm).await?;
self.setup_start(ctx, opt, stm).await?;
// Process any EXPLAIN clause
let explanation = self.output_explain(ctx, opt, stm)?;
// Process prepared values
self.iterate(&run, opt, txn, stm).await?;
self.iterate(ctx, opt, stm).await?;
// Return any document errors
if let Some(e) = self.error.take() {
return Err(e);
}
// Process any SPLIT clause
self.output_split(ctx, opt, txn, stm).await?;
self.output_split(ctx, opt, stm).await?;
// Process any GROUP clause
self.output_group(ctx, opt, txn, stm).await?;
self.output_group(ctx, opt, stm).await?;
// Process any ORDER clause
self.output_order(ctx, opt, txn, stm).await?;
self.output_order(ctx, opt, stm).await?;
// Process any START clause
self.output_start(ctx, opt, txn, stm).await?;
self.output_start(ctx, opt, stm).await?;
// Process any LIMIT clause
self.output_limit(ctx, opt, txn, stm).await?;
self.output_limit(ctx, opt, stm).await?;
// Process any FETCH clause
self.output_fetch(ctx, opt, txn, stm).await?;
self.output_fetch(ctx, opt, stm).await?;
// Add the EXPLAIN clause to the result
if let Some(e) = explanation {
self.results.push(e);
}
// Output the results
Ok(mem::take(&mut self.results).into())
}
@ -112,11 +119,10 @@ impl Iterator {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<(), Error> {
if let Some(v) = stm.limit() {
self.limit = Some(v.process(ctx, opt, txn, None).await?);
self.limit = Some(v.process(ctx, opt).await?);
}
Ok(())
}
@ -126,11 +132,10 @@ impl Iterator {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<(), Error> {
if let Some(v) = stm.start() {
self.start = Some(v.process(ctx, opt, txn, None).await?);
self.start = Some(v.process(ctx, opt).await?);
}
Ok(())
}
@ -140,7 +145,6 @@ impl Iterator {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<(), Error> {
if let Some(splits) = stm.split() {
@ -159,7 +163,7 @@ impl Iterator {
// Make a copy of object
let mut obj = obj.clone();
// Set the value at the path
obj.set(ctx, opt, txn, split, val).await?;
obj.set(ctx, opt, split, val).await?;
// Add the object to the results
self.results.push(obj);
}
@ -168,7 +172,7 @@ impl Iterator {
// Make a copy of object
let mut obj = obj.clone();
// Set the value at the path
obj.set(ctx, opt, txn, split, val).await?;
obj.set(ctx, opt, split, val).await?;
// Add the object to the results
self.results.push(obj);
}
@ -184,7 +188,6 @@ impl Iterator {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<(), Error> {
if let Some(fields) = stm.expr() {
@ -232,19 +235,20 @@ impl Iterator {
.unwrap_or_else(|| Cow::Owned(expr.to_idiom()));
match expr {
Value::Function(f) if f.is_aggregate() => {
let x =
vals.all().get(ctx, opt, txn, None, idiom.as_ref()).await?;
let x = f.aggregate(x).compute(ctx, opt, txn, None).await?;
obj.set(ctx, opt, txn, idiom.as_ref(), x).await?;
let x = vals.all().get(ctx, opt, idiom.as_ref()).await?;
let x = f.aggregate(x).compute(ctx, opt).await?;
obj.set(ctx, opt, idiom.as_ref(), x).await?;
}
_ => {
let x = vals.first();
let mut child_ctx = Context::new(ctx);
child_ctx.add_cursor_doc(&x);
let x = if let Some(alias) = alias {
alias.compute(ctx, opt, txn, Some(&x)).await?
alias.compute(&child_ctx, opt).await?
} else {
expr.compute(ctx, opt, txn, Some(&x)).await?
expr.compute(&child_ctx, opt).await?
};
obj.set(ctx, opt, txn, idiom.as_ref(), x).await?;
obj.set(ctx, opt, idiom.as_ref(), x).await?;
}
}
}
@ -262,7 +266,6 @@ impl Iterator {
&mut self,
_ctx: &Context<'_>,
_opt: &Options,
_txn: &Transaction,
stm: &Statement<'_>,
) -> Result<(), Error> {
if let Some(orders) = stm.order() {
@ -301,7 +304,6 @@ impl Iterator {
&mut self,
_ctx: &Context<'_>,
_opt: &Options,
_txn: &Transaction,
_stm: &Statement<'_>,
) -> Result<(), Error> {
if let Some(v) = self.start {
@ -315,7 +317,6 @@ impl Iterator {
&mut self,
_ctx: &Context<'_>,
_opt: &Options,
_txn: &Transaction,
_stm: &Statement<'_>,
) -> Result<(), Error> {
if let Some(v) = self.limit {
@ -329,7 +330,6 @@ impl Iterator {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<(), Error> {
if let Some(fetchs) = stm.fetch() {
@ -337,27 +337,82 @@ impl Iterator {
// Loop over each result value
for obj in &mut self.results {
// Fetch the value at the path
obj.fetch(ctx, opt, txn, fetch).await?;
obj.fetch(ctx, opt, fetch).await?;
}
}
}
Ok(())
}
#[inline]
fn output_explain(
&mut self,
_ctx: &Context<'_>,
_opt: &Options,
stm: &Statement<'_>,
) -> Result<Option<Value>, Error> {
Ok(if stm.explain() {
let mut explains = Vec::with_capacity(self.entries.len());
for iter in &self.entries {
let (operation, detail) = match iter {
Iterable::Value(v) => ("Iterate Value", vec![("value", v.to_owned())]),
Iterable::Table(t) => {
("Iterate Table", vec![("table", Value::from(t.0.to_owned()))])
}
Iterable::Thing(t) => {
("Iterate Thing", vec![("thing", Value::Thing(t.to_owned()))])
}
Iterable::Range(r) => {
("Iterate Range", vec![("table", Value::from(r.tb.to_owned()))])
}
Iterable::Edges(e) => {
("Iterate Edges", vec![("from", Value::Thing(e.from.to_owned()))])
}
Iterable::Mergeable(t, v) => (
"Iterate Mergeable",
vec![("thing", Value::Thing(t.to_owned())), ("value", v.to_owned())],
),
Iterable::Relatable(t1, t2, t3) => (
"Iterate Relatable",
vec![
("thing-1", Value::Thing(t1.to_owned())),
("thing-2", Value::Thing(t2.to_owned())),
("thing-3", Value::Thing(t3.to_owned())),
],
),
Iterable::Index(t, p) => (
"Iterate Index",
vec![("table", Value::from(t.0.to_owned())), ("plan", p.explain())],
),
};
let explain = Object::from(HashMap::from([
("operation", Value::from(operation)),
("detail", Value::Object(Object::from(HashMap::from_iter(detail)))),
]));
explains.push(Value::Object(explain));
}
Some(Value::Object(Object::from(HashMap::from([(
"explain",
Value::Array(Array::from(explains)),
)]))))
} else {
None
})
}
#[cfg(target_arch = "wasm32")]
#[async_recursion(?Send)]
async fn iterate(
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<(), Error> {
// Prevent deep recursion
let opt = &opt.dive(4)?;
// Process all prepared values
for v in mem::take(&mut self.entries) {
v.iterate(ctx, opt, txn, stm, self).await?;
v.iterate(ctx, opt, stm, self).await?;
}
// Everything processed ok
Ok(())
@ -369,7 +424,6 @@ impl Iterator {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<(), Error> {
// Prevent deep recursion
@ -380,7 +434,7 @@ impl Iterator {
false => {
// Process all prepared values
for v in mem::take(&mut self.entries) {
v.iterate(ctx, opt, txn, stm, self).await?;
v.iterate(ctx, opt, stm, self).await?;
}
// Everything processed ok
Ok(())
@ -388,7 +442,7 @@ impl Iterator {
// Run statements in parallel
true => {
// Create a new executor
let exe = executor::Executor::new();
let e = executor::Executor::new();
// Take all of the iterator values
let vals = mem::take(&mut self.entries);
// Create a channel to shutdown
@ -399,7 +453,7 @@ impl Iterator {
let adocs = async {
// Process all prepared values
for v in vals {
exe.spawn(v.channel(ctx, opt, txn, stm, chn.clone()))
e.spawn(v.channel(ctx, opt, stm, chn.clone()))
// Ensure we detach the spawned task
.detach();
}
@ -412,7 +466,7 @@ impl Iterator {
let avals = async {
// Process all received values
while let Ok((k, v)) = docs.recv().await {
exe.spawn(Document::compute(ctx, opt, txn, stm, chn.clone(), k, v))
e.spawn(Document::compute(ctx, opt, stm, chn.clone(), k, v))
// Ensure we detach the spawned task
.detach();
}
@ -429,7 +483,7 @@ impl Iterator {
let _ = end.send(()).await;
};
// Run all executor tasks
let fut = exe.run(exit.recv());
let fut = e.run(exit.recv());
// Wait for all closures
let res = futures::join!(adocs, avals, aproc, fut);
// Consume executor error
@ -445,9 +499,7 @@ impl Iterator {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
thg: Option<Thing>,
val: Operable,
) {
// Check current context
@ -455,21 +507,21 @@ impl Iterator {
return;
}
// Setup a new workable
let val = match val {
let (val, ext) = match val {
Operable::Value(v) => (v, Workable::Normal),
Operable::Mergeable(v, o) => (v, Workable::Insert(o)),
Operable::Relatable(f, v, w) => (v, Workable::Relate(f, w)),
};
// Setup a new document
let mut doc = Document::new(thg, &val.0, val.1);
let mut doc = Document::new(ctx.thing(), &val, ext);
// Process the document
let res = match stm {
Statement::Select(_) => doc.select(ctx, opt, txn, stm).await,
Statement::Create(_) => doc.create(ctx, opt, txn, stm).await,
Statement::Update(_) => doc.update(ctx, opt, txn, stm).await,
Statement::Relate(_) => doc.relate(ctx, opt, txn, stm).await,
Statement::Delete(_) => doc.delete(ctx, opt, txn, stm).await,
Statement::Insert(_) => doc.insert(ctx, opt, txn, stm).await,
Statement::Select(_) => doc.select(ctx, opt, stm).await,
Statement::Create(_) => doc.create(ctx, opt, stm).await,
Statement::Update(_) => doc.update(ctx, opt, stm).await,
Statement::Relate(_) => doc.relate(ctx, opt, stm).await,
Statement::Delete(_) => doc.delete(ctx, opt, stm).await,
Statement::Insert(_) => doc.insert(ctx, opt, stm).await,
_ => unreachable!(),
};
// Process the result

8
lib/src/dbs/statement.rs
View file

@ -200,4 +200,12 @@ impl<'a> Statement<'a> {
_ => false,
}
}
/// Returns any EXPLAIN clause if specified
#[inline]
pub fn explain(&self) -> bool {
match self {
Statement::Select(v) => v.explain,
_ => false,
}
}
}

8
lib/src/dbs/test.rs
View file

@ -1,15 +1,15 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::kvs::Datastore;
use futures::lock::Mutex;
use std::sync::Arc;
pub async fn mock<'a>() -> (Context<'a>, Options, Transaction) {
let ctx = Context::default();
pub async fn mock<'a>() -> (Context<'a>, Options) {
let mut ctx = Context::default();
let opt = Options::default();
let kvs = Datastore::new("memory").await.unwrap();
let txn = kvs.transaction(true, false).await.unwrap();
let txn = Arc::new(Mutex::new(txn));
(ctx, opt, txn)
ctx.add_transaction(Some(&txn));
(ctx, opt)
}

10
lib/src/doc/allow.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
use crate::sql::permission::Permission;
@ -11,15 +10,16 @@ impl<'a> Document<'a> {
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<(), Error> {
// Check if this record exists
if self.id.is_some() {
// Should we run permissions checks?
if opt.perms && opt.auth.perms() {
// Clone transaction
let txn = ctx.clone_transaction()?;
// Get the table
let tb = self.tb(opt, txn).await?;
let tb = self.tb(opt, &txn).await?;
// Get the permission clause
let perms = if stm.is_delete() {
&tb.permissions.delete
@ -37,8 +37,10 @@ impl<'a> Document<'a> {
Permission::Specific(e) => {
// Disable permissions
let opt = &opt.perms(false);
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(&self.current);
// Process the PERMISSION clause
if !e.compute(ctx, opt, txn, Some(&self.current)).await?.is_truthy() {
if !e.compute(&ctx, opt).await?.is_truthy() {
return Err(Error::Ignore);
}
}

52
lib/src/doc/alter.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::dbs::Workable;
use crate::doc::Document;
use crate::err::Error;
@ -14,7 +13,6 @@ impl<'a> Document<'a> {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<(), Error> {
// Get the record id
@ -25,39 +23,47 @@ impl<'a> Document<'a> {
if let Some(v) = stm.data() {
match v {
Data::PatchExpression(data) => {
let data = data.compute(ctx, opt, txn, Some(&self.current)).await?;
let mut current_ctx = Context::new(ctx);
current_ctx.add_cursor_doc(&self.current);
let data = data.compute(&current_ctx, opt).await?;
self.current.to_mut().patch(data)?
}
Data::MergeExpression(data) => {
let data = data.compute(ctx, opt, txn, Some(&self.current)).await?;
let mut current_ctx = Context::new(ctx);
current_ctx.add_cursor_doc(&self.current);
let data = data.compute(&current_ctx, opt).await?;
self.current.to_mut().merge(data)?
}
Data::ReplaceExpression(data) => {
let data = data.compute(ctx, opt, txn, Some(&self.current)).await?;
let mut current_ctx = Context::new(ctx);
current_ctx.add_cursor_doc(&self.current);
let data = data.compute(&current_ctx, opt).await?;
self.current.to_mut().replace(data)?
}
Data::ContentExpression(data) => {
let data = data.compute(ctx, opt, txn, Some(&self.current)).await?;
let mut current_ctx = Context::new(ctx);
current_ctx.add_cursor_doc(&self.current);
let data = data.compute(&current_ctx, opt).await?;
self.current.to_mut().replace(data)?
}
Data::SetExpression(x) => {
for x in x.iter() {
let v = x.2.compute(ctx, opt, txn, Some(&self.current)).await?;
let mut current_ctx = Context::new(ctx);
current_ctx.add_cursor_doc(&self.current);
let v = x.2.compute(&current_ctx, opt).await?;
match x.1 {
Operator::Equal => match v {
Value::None => {
self.current.to_mut().del(ctx, opt, txn, &x.0).await?
}
_ => self.current.to_mut().set(ctx, opt, txn, &x.0, v).await?,
Value::None => self.current.to_mut().del(ctx, opt, &x.0).await?,
_ => self.current.to_mut().set(ctx, opt, &x.0, v).await?,
},
Operator::Inc => {
self.current.to_mut().increment(ctx, opt, txn, &x.0, v).await?
self.current.to_mut().increment(ctx, opt, &x.0, v).await?
}
Operator::Dec => {
self.current.to_mut().decrement(ctx, opt, txn, &x.0, v).await?
self.current.to_mut().decrement(ctx, opt, &x.0, v).await?
}
Operator::Ext => {
self.current.to_mut().extend(ctx, opt, txn, &x.0, v).await?
self.current.to_mut().extend(ctx, opt, &x.0, v).await?
}
_ => unreachable!(),
}
@ -65,7 +71,7 @@ impl<'a> Document<'a> {
}
Data::UnsetExpression(i) => {
for i in i.iter() {
self.current.to_mut().del(ctx, opt, txn, &i).await?
self.current.to_mut().del(ctx, opt, &i).await?
}
}
Data::UpdateExpression(x) => {
@ -77,22 +83,22 @@ impl<'a> Document<'a> {
}
// Process ON DUPLICATE KEY clause
for x in x.iter() {
let v = x.2.compute(&ctx, opt, txn, Some(&self.current)).await?;
let mut current_ctx = Context::new(&ctx);
current_ctx.add_cursor_doc(&self.current);
let v = x.2.compute(&current_ctx, opt).await?;
match x.1 {
Operator::Equal => match v {
Value::None => {
self.current.to_mut().del(&ctx, opt, txn, &x.0).await?
}
_ => self.current.to_mut().set(&ctx, opt, txn, &x.0, v).await?,
Value::None => self.current.to_mut().del(&ctx, opt, &x.0).await?,
_ => self.current.to_mut().set(&ctx, opt, &x.0, v).await?,
},
Operator::Inc => {
self.current.to_mut().increment(&ctx, opt, txn, &x.0, v).await?
self.current.to_mut().increment(&ctx, opt, &x.0, v).await?
}
Operator::Dec => {
self.current.to_mut().decrement(&ctx, opt, txn, &x.0, v).await?
self.current.to_mut().decrement(&ctx, opt, &x.0, v).await?
}
Operator::Ext => {
self.current.to_mut().extend(&ctx, opt, txn, &x.0, v).await?
self.current.to_mut().extend(&ctx, opt, &x.0, v).await?
}
_ => unreachable!(),
}

6
lib/src/doc/check.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
@ -10,13 +9,14 @@ impl<'a> Document<'a> {
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<(), Error> {
// Check where condition
if let Some(cond) = stm.conds() {
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(&self.current);
// Check if the expression is truthy
if !cond.compute(ctx, opt, txn, Some(&self.current)).await?.is_truthy() {
if !cond.compute(&ctx, opt).await?.is_truthy() {
// Ignore this document
return Err(Error::Ignore);
}

10
lib/src/doc/clean.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
use crate::sql::idiom::Idiom;
@ -11,17 +10,18 @@ impl<'a> Document<'a> {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_stm: &Statement<'_>,
) -> Result<(), Error> {
// Clone transaction
let txn = ctx.clone_transaction()?;
// Get the table
let tb = self.tb(opt, txn).await?;
let tb = self.tb(opt, &txn).await?;
// This table is schemafull
if tb.full {
// Create a vector to store the keys
let mut keys: Vec<Idiom> = vec![];
// Loop through all field statements
for fd in self.fd(opt, txn).await?.iter() {
for fd in self.fd(opt, &txn).await?.iter() {
// Is this a schemaless field?
match fd.flex {
false => {
@ -46,7 +46,7 @@ impl<'a> Document<'a> {
fd if fd.is_in() => continue,
fd if fd.is_out() => continue,
fd if fd.is_meta() => continue,
fd => self.current.to_mut().del(ctx, opt, txn, fd).await?,
fd => self.current.to_mut().del(ctx, opt, fd).await?,
}
}
}

20
lib/src/doc/compute.rs
View file

@ -2,7 +2,6 @@ use crate::ctx::Context;
use crate::dbs::Operable;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::dbs::Workable;
use crate::doc::Document;
use crate::err::Error;
@ -15,12 +14,15 @@ impl<'a> Document<'a> {
pub(crate) async fn compute(
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
chn: Sender<Result<Value, Error>>,
thg: Option<Thing>,
val: Operable,
) -> Result<(), Error> {
let mut ctx = Context::new(ctx);
if let Some(t) = &thg {
ctx.add_thing(t);
}
// Setup a new workable
let ins = match val {
Operable::Value(v) => (v, Workable::Normal),
@ -28,15 +30,15 @@ impl<'a> Document<'a> {
Operable::Relatable(f, v, w) => (v, Workable::Relate(f, w)),
};
// Setup a new document
let mut doc = Document::new(thg, &ins.0, ins.1);
let mut doc = Document::new(ctx.thing(), &ins.0, ins.1);
// Process the statement
let res = match stm {
Statement::Select(_) => doc.select(ctx, opt, txn, stm).await,
Statement::Create(_) => doc.create(ctx, opt, txn, stm).await,
Statement::Update(_) => doc.update(ctx, opt, txn, stm).await,
Statement::Relate(_) => doc.relate(ctx, opt, txn, stm).await,
Statement::Delete(_) => doc.delete(ctx, opt, txn, stm).await,
Statement::Insert(_) => doc.insert(ctx, opt, txn, stm).await,
Statement::Select(_) => doc.select(&ctx, opt, stm).await,
Statement::Create(_) => doc.create(&ctx, opt, stm).await,
Statement::Update(_) => doc.update(&ctx, opt, stm).await,
Statement::Relate(_) => doc.relate(&ctx, opt, stm).await,
Statement::Delete(_) => doc.delete(&ctx, opt, stm).await,
Statement::Insert(_) => doc.insert(&ctx, opt, stm).await,
_ => unreachable!(),
};
// Send back the result

26
lib/src/doc/create.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
use crate::sql::value::Value;
@ -11,32 +10,31 @@ impl<'a> Document<'a> {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<Value, Error> {
// Check if exists
self.exist(ctx, opt, txn, stm).await?;
self.exist(ctx, opt, stm).await?;
// Alter record data
self.alter(ctx, opt, txn, stm).await?;
self.alter(ctx, opt, stm).await?;
// Merge fields data
self.field(ctx, opt, txn, stm).await?;
self.field(ctx, opt, stm).await?;
// Reset fields data
self.reset(ctx, opt, txn, stm).await?;
self.reset(ctx, opt, stm).await?;
// Clean fields data
self.clean(ctx, opt, txn, stm).await?;
self.clean(ctx, opt, stm).await?;
// Check if allowed
self.allow(ctx, opt, txn, stm).await?;
self.allow(ctx, opt, stm).await?;
// Store index data
self.index(ctx, opt, txn, stm).await?;
self.index(ctx, opt, stm).await?;
// Store record data
self.store(ctx, opt, txn, stm).await?;
self.store(ctx, opt, stm).await?;
// Run table queries
self.table(ctx, opt, txn, stm).await?;
self.table(ctx, opt, stm).await?;
// Run lives queries
self.lives(ctx, opt, txn, stm).await?;
self.lives(ctx, opt, stm).await?;
// Run event queries
self.event(ctx, opt, txn, stm).await?;
self.event(ctx, opt, stm).await?;
// Yield document
self.pluck(ctx, opt, txn, stm).await
self.pluck(ctx, opt, stm).await
}
}

20
lib/src/doc/delete.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
use crate::sql::value::Value;
@ -11,26 +10,25 @@ impl<'a> Document<'a> {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<Value, Error> {
// Check where clause
self.check(ctx, opt, txn, stm).await?;
self.check(ctx, opt, stm).await?;
// Check if allowed
self.allow(ctx, opt, txn, stm).await?;
self.allow(ctx, opt, stm).await?;
// Erase document
self.erase(ctx, opt, txn, stm).await?;
self.erase(ctx, opt, stm).await?;
// Purge index data
self.index(ctx, opt, txn, stm).await?;
self.index(ctx, opt, stm).await?;
// Purge record data
self.purge(ctx, opt, txn, stm).await?;
self.purge(ctx, opt, stm).await?;
// Run table queries
self.table(ctx, opt, txn, stm).await?;
self.table(ctx, opt, stm).await?;
// Run lives queries
self.lives(ctx, opt, txn, stm).await?;
self.lives(ctx, opt, stm).await?;
// Run event queries
self.event(ctx, opt, txn, stm).await?;
self.event(ctx, opt, stm).await?;
// Yield document
self.pluck(ctx, opt, txn, stm).await
self.pluck(ctx, opt, stm).await
}
}

4
lib/src/doc/document.rs
View file

@ -15,7 +15,7 @@ use std::fmt::{Debug, Formatter};
use std::sync::Arc;
pub(crate) struct Document<'a> {
pub(super) id: Option<Thing>,
pub(super) id: Option<&'a Thing>,
pub(super) extras: Workable,
pub(super) current: Cow<'a, Value>,
pub(super) initial: Cow<'a, Value>,
@ -34,7 +34,7 @@ impl<'a> From<&Document<'a>> for Vec<u8> {
}
impl<'a> Document<'a> {
pub fn new(id: Option<Thing>, val: &'a Value, ext: Workable) -> Self {
pub fn new(id: Option<&'a Thing>, val: &'a Value, ext: Workable) -> Self {
Document {
id,
extras: ext,

12
lib/src/doc/edges.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::dbs::Workable;
use crate::doc::Document;
use crate::err::Error;
@ -14,19 +13,18 @@ use crate::sql::Dir;
impl<'a> Document<'a> {
pub async fn edges(
&mut self,
_ctx: &Context<'_>,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_stm: &Statement<'_>,
) -> Result<(), Error> {
// Clone transaction
let txn = ctx.clone_transaction()?;
// Check if the table is a view
if self.tb(opt, txn).await?.drop {
if self.tb(opt, &txn).await?.drop {
return Ok(());
}
// Clone transaction
let run = txn.clone();
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Get the record id
let rid = self.id.as_ref().unwrap();
// Store the record edges

2
lib/src/doc/empty.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
@ -10,7 +9,6 @@ impl<'a> Document<'a> {
&self,
_ctx: &Context<'_>,
_opt: &Options,
_txn: &Transaction,
_stm: &Statement<'_>,
) -> Result<(), Error> {
// Check if this record exists

2
lib/src/doc/erase.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
@ -10,7 +9,6 @@ impl<'a> Document<'a> {
&mut self,
_ctx: &Context<'_>,
_opt: &Options,
_txn: &Transaction,
_stm: &Statement<'_>,
) -> Result<(), Error> {
self.current.to_mut().clear()

11
lib/src/doc/event.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
use crate::sql::value::Value;
@ -12,7 +11,6 @@ impl<'a> Document<'a> {
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<(), Error> {
// Check events
@ -25,8 +23,10 @@ impl<'a> Document<'a> {
}
// Don't run permissions
let opt = &opt.perms(false);
// Clone transaction
let txn = ctx.clone_transaction()?;
// Loop through all event statements
for ev in self.ev(opt, txn).await?.iter() {
for ev in self.ev(opt, &txn).await?.iter() {
// Get the event action
let met = if stm.is_delete() {
Value::from("DELETE")
@ -41,12 +41,13 @@ impl<'a> Document<'a> {
ctx.add_value("value", self.current.deref());
ctx.add_value("after", self.current.deref());
ctx.add_value("before", self.initial.deref());
ctx.add_cursor_doc(&self.current);
// Process conditional clause
let val = ev.when.compute(&ctx, opt, txn, Some(&self.current)).await?;
let val = ev.when.compute(&ctx, opt).await?;
// Execute event if value is truthy
if val.is_truthy() {
for v in ev.then.iter() {
v.compute(&ctx, opt, txn, Some(&self.current)).await?;
v.compute(&ctx, opt).await?;
}
}
}

2
lib/src/doc/exist.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
@ -10,7 +9,6 @@ impl<'a> Document<'a> {
&self,
_ctx: &Context<'_>,
_opt: &Options,
_txn: &Transaction,
_stm: &Statement<'_>,
) -> Result<(), Error> {
// Check if this record exists

19
lib/src/doc/field.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
use crate::sql::permission::Permission;
@ -12,7 +11,6 @@ impl<'a> Document<'a> {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_stm: &Statement<'_>,
) -> Result<(), Error> {
// Check fields
@ -23,8 +21,10 @@ impl<'a> Document<'a> {
let rid = self.id.as_ref().unwrap();
// Get the user applied input
let inp = self.initial.changed(self.current.as_ref());
// Clone transaction
let txn = ctx.clone_transaction()?;
// Loop through all field statements
for fd in self.fd(opt, txn).await?.iter() {
for fd in self.fd(opt, &txn).await?.iter() {
// Loop over each field in document
for (k, mut val) in self.current.walk(&fd.name).into_iter() {
// Get the initial value
@ -58,8 +58,9 @@ impl<'a> Document<'a> {
ctx.add_value("value", &val);
ctx.add_value("after", &val);
ctx.add_value("before", &old);
ctx.add_cursor_doc(&self.current);
// Process the VALUE clause
val = expr.compute(&ctx, opt, txn, Some(&self.current)).await?;
val = expr.compute(&ctx, opt).await?;
}
// Check for a TYPE clause
if let Some(kind) = &fd.kind {
@ -86,8 +87,9 @@ impl<'a> Document<'a> {
ctx.add_value("value", &val);
ctx.add_value("after", &val);
ctx.add_value("before", &old);
ctx.add_cursor_doc(&self.current);
// Process the ASSERT clause
if !expr.compute(&ctx, opt, txn, Some(&self.current)).await?.is_truthy() {
if !expr.compute(&ctx, opt).await?.is_truthy() {
return Err(Error::FieldValue {
thing: rid.to_string(),
field: fd.name.clone(),
@ -117,8 +119,9 @@ impl<'a> Document<'a> {
ctx.add_value("value", &val);
ctx.add_value("after", &val);
ctx.add_value("before", &old);
ctx.add_cursor_doc(&self.current);
// Process the PERMISSION clause
if !e.compute(&ctx, opt, txn, Some(&self.current)).await?.is_truthy() {
if !e.compute(&ctx, opt).await?.is_truthy() {
val = old
}
}
@ -126,8 +129,8 @@ impl<'a> Document<'a> {
}
// Set the value of the field
match val {
Value::None => self.current.to_mut().del(ctx, opt, txn, &k).await?,
_ => self.current.to_mut().set(ctx, opt, txn, &k, val).await?,
Value::None => self.current.to_mut().del(ctx, opt, &k).await?,
_ => self.current.to_mut().set(ctx, opt, &k, val).await?,
};
}
}

45
lib/src/doc/index.rs
View file

@ -1,17 +1,15 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
// use crate::idx::ft::FtIndex;
use crate::idx::ft::FtIndex;
use crate::idx::IndexKeyBase;
use crate::sql::array::Array;
use crate::sql::index::Index;
use crate::sql::scoring::Scoring;
use crate::sql::statements::DefineIndexStatement;
use crate::sql::{Ident, Number, Thing, Value};
use crate::sql::{Ident, Thing, Value};
use crate::{key, kvs};
impl<'a> Document<'a> {
@ -19,7 +17,6 @@ impl<'a> Document<'a> {
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_stm: &Statement<'_>,
) -> Result<(), Error> {
// Check events
@ -30,19 +27,21 @@ impl<'a> Document<'a> {
if !opt.force && !self.changed() {
return Ok(());
}
// Clone transaction
let txn = ctx.clone_transaction()?;
// Check if the table is a view
if self.tb(opt, txn).await?.drop {
if self.tb(opt, &txn).await?.drop {
return Ok(());
}
// Get the record id
let rid = self.id.as_ref().unwrap();
// Loop through all index statements
for ix in self.ix(opt, txn).await?.iter() {
for ix in self.ix(opt, &txn).await?.iter() {
// Calculate old values
let o = Self::build_opt_array(ctx, &txn, opt, ix, &self.initial).await?;
let o = Self::build_opt_array(ctx, opt, ix, &self.initial).await?;
// Calculate new values
let n = Self::build_opt_array(ctx, &txn, opt, ix, &self.current).await?;
let n = Self::build_opt_array(ctx, opt, ix, &self.current).await?;
// Update the index entries
if opt.force || o != n {
@ -60,12 +59,11 @@ impl<'a> Document<'a> {
az,
sc,
hl,
order,
} => match sc {
Scoring::Bm {
k1,
b,
order,
} => ic.index_best_matching_search(&mut run, az, k1, b, order, *hl).await?,
..
} => ic.index_best_matching_search(&mut run, az, *order, *hl).await?,
Scoring::Vs => ic.index_vector_search(az, *hl).await?,
},
};
@ -75,9 +73,12 @@ impl<'a> Document<'a> {
Ok(())
}
/// Extract from the given document, the values required by the index and put then in an array.
/// Eg. IF the index is composed of the columns `name` and `instrument`
/// Given this doc: { "id": 1, "instrument":"piano", "name":"Tobie" }
/// It will return: ["Tobie", "piano"]
async fn build_opt_array(
ctx: &Context<'_>,
txn: &Transaction,
opt: &Options,
ix: &DefineIndexStatement,
value: &Value,
@ -85,9 +86,11 @@ impl<'a> Document<'a> {
if !value.is_some() {
return Ok(None);
}
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(value);
let mut o = Array::with_capacity(ix.cols.len());
for i in ix.cols.iter() {
let v = i.compute(ctx, opt, txn, Some(value)).await?;
let v = i.compute(&ctx, opt).await?;
o.push(v);
}
Ok(Some(o))
@ -182,20 +185,18 @@ impl<'a> IndexOperation<'a> {
async fn index_best_matching_search(
&self,
run: &mut kvs::Transaction,
_az: &Ident,
_k1: &Number,
_b: &Number,
order: &Number,
az: &Ident,
order: u32,
_hl: bool,
) -> Result<(), Error> {
let ikb = IndexKeyBase::new(self.opt, self.ix);
let mut ft = FtIndex::new(run, ikb, order.to_usize()).await?;
let doc_key = self.rid.into();
let az = run.get_az(self.opt.ns(), self.opt.db(), az.as_str()).await?;
let mut ft = FtIndex::new(run, az, ikb, order).await?;
if let Some(n) = &self.n {
// TODO: Apply the analyzer
ft.index_document(run, doc_key, &n.to_string()).await
ft.index_document(run, self.rid, n).await
} else {
ft.remove_document(run, doc_key).await
ft.remove_document(run, self.rid).await
}
}

50
lib/src/doc/insert.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
use crate::sql::value::Value;
@ -11,7 +10,6 @@ impl<'a> Document<'a> {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<Value, Error> {
// Check current record
@ -19,56 +17,56 @@ impl<'a> Document<'a> {
// Run INSERT clause
false => {
// Check if allowed
self.allow(ctx, opt, txn, stm).await?;
self.allow(ctx, opt, stm).await?;
// Merge record data
self.merge(ctx, opt, txn, stm).await?;
self.merge(ctx, opt, stm).await?;
// Merge fields data
self.field(ctx, opt, txn, stm).await?;
self.field(ctx, opt, stm).await?;
// Reset fields data
self.reset(ctx, opt, txn, stm).await?;
self.reset(ctx, opt, stm).await?;
// Clean fields data
self.clean(ctx, opt, txn, stm).await?;
self.clean(ctx, opt, stm).await?;
// Check if allowed
self.allow(ctx, opt, txn, stm).await?;
self.allow(ctx, opt, stm).await?;
// Store index data
self.index(ctx, opt, txn, stm).await?;
self.index(ctx, opt, stm).await?;
// Store record data
self.store(ctx, opt, txn, stm).await?;
self.store(ctx, opt, stm).await?;
// Run table queries
self.table(ctx, opt, txn, stm).await?;
self.table(ctx, opt, stm).await?;
// Run lives queries
self.lives(ctx, opt, txn, stm).await?;
self.lives(ctx, opt, stm).await?;
// Run event queries
self.event(ctx, opt, txn, stm).await?;
self.event(ctx, opt, stm).await?;
// Yield document
self.pluck(ctx, opt, txn, stm).await
self.pluck(ctx, opt, stm).await
}
// Run UPDATE clause
true => {
// Check if allowed
self.allow(ctx, opt, txn, stm).await?;
self.allow(ctx, opt, stm).await?;
// Alter record data
self.alter(ctx, opt, txn, stm).await?;
self.alter(ctx, opt, stm).await?;
// Merge fields data
self.field(ctx, opt, txn, stm).await?;
self.field(ctx, opt, stm).await?;
// Reset fields data
self.reset(ctx, opt, txn, stm).await?;
self.reset(ctx, opt, stm).await?;
// Clean fields data
self.clean(ctx, opt, txn, stm).await?;
self.clean(ctx, opt, stm).await?;
// Check if allowed
self.allow(ctx, opt, txn, stm).await?;
self.allow(ctx, opt, stm).await?;
// Store index data
self.index(ctx, opt, txn, stm).await?;
self.index(ctx, opt, stm).await?;
// Store record data
self.store(ctx, opt, txn, stm).await?;
self.store(ctx, opt, stm).await?;
// Run table queries
self.table(ctx, opt, txn, stm).await?;
self.table(ctx, opt, stm).await?;
// Run lives queries
self.lives(ctx, opt, txn, stm).await?;
self.lives(ctx, opt, stm).await?;
// Run event queries
self.event(ctx, opt, txn, stm).await?;
self.event(ctx, opt, stm).await?;
// Yield document
self.pluck(ctx, opt, txn, stm).await
self.pluck(ctx, opt, stm).await
}
}
}

12
lib/src/doc/lives.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
@ -10,21 +9,22 @@ impl<'a> Document<'a> {
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_stm: &Statement<'_>,
) -> Result<(), Error> {
// Check if forced
if !opt.force && !self.changed() {
return Ok(());
}
// Clone transaction
let txn = ctx.clone_transaction()?;
// Get the record id
let _ = self.id.as_ref().unwrap();
// Loop through all index statements
for lv in self.lv(opt, txn).await?.iter() {
for lv in self.lv(opt, &txn).await?.iter() {
// Create a new statement
let stm = Statement::from(lv);
// Check LIVE SELECT where condition
if self.check(ctx, opt, txn, &stm).await.is_err() {
if self.check(ctx, opt, &stm).await.is_err() {
continue;
}
// Check what type of data change this is
@ -32,10 +32,10 @@ impl<'a> Document<'a> {
// Send a DELETE notification to the WebSocket
} else if self.is_new() {
// Process the CREATE notification to send
let _ = self.pluck(ctx, opt, txn, &stm).await?;
let _ = self.pluck(ctx, opt, &stm).await?;
} else {
// Process the CREATE notification to send
let _ = self.pluck(ctx, opt, txn, &stm).await?;
let _ = self.pluck(ctx, opt, &stm).await?;
};
}
// Carry on

6
lib/src/doc/merge.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::dbs::Workable;
use crate::doc::Document;
use crate::err::Error;
@ -11,7 +10,6 @@ impl<'a> Document<'a> {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_stm: &Statement<'_>,
) -> Result<(), Error> {
// Get the record id
@ -20,7 +18,9 @@ impl<'a> Document<'a> {
self.current.to_mut().def(rid);
// This is an INSERT statement
if let Workable::Insert(v) = &self.extras {
let v = v.compute(ctx, opt, txn, Some(&self.current)).await?;
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(&self.current);
let v = v.compute(&ctx, opt).await?;
self.current.to_mut().merge(v)?;
}
// Set default field values

63
lib/src/doc/pluck.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
use crate::sql::idiom::Idiom;
@ -15,7 +14,6 @@ impl<'a> Document<'a> {
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<Value, Error> {
// Ensure futures are run
@ -26,29 +24,55 @@ impl<'a> Document<'a> {
Output::None => Err(Error::Ignore),
Output::Null => Ok(Value::Null),
Output::Diff => Ok(self.initial.diff(&self.current, Idiom::default()).into()),
Output::After => self.current.compute(ctx, opt, txn, Some(&self.current)).await,
Output::Before => self.initial.compute(ctx, opt, txn, Some(&self.initial)).await,
Output::Fields(v) => v.compute(ctx, opt, txn, Some(&self.current), false).await,
Output::After => {
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(&self.current);
self.current.compute(&ctx, opt).await
}
Output::Before => {
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(&self.initial);
self.initial.compute(&ctx, opt).await
}
Output::Fields(v) => {
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(&self.current);
v.compute(&ctx, opt, false).await
}
},
None => match stm {
Statement::Live(s) => match s.expr.len() {
0 => Ok(self.initial.diff(&self.current, Idiom::default()).into()),
_ => s.expr.compute(ctx, opt, txn, Some(&self.current), false).await,
_ => {
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(&self.current);
s.expr.compute(&ctx, opt, false).await
}
},
Statement::Select(s) => {
s.expr.compute(ctx, opt, txn, Some(&self.current), s.group.is_some()).await
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(&self.current);
s.expr.compute(&ctx, opt, s.group.is_some()).await
}
Statement::Create(_) => {
self.current.compute(ctx, opt, txn, Some(&self.current)).await
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(&self.current);
self.current.compute(&ctx, opt).await
}
Statement::Update(_) => {
self.current.compute(ctx, opt, txn, Some(&self.current)).await
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(&self.current);
self.current.compute(&ctx, opt).await
}
Statement::Relate(_) => {
self.current.compute(ctx, opt, txn, Some(&self.current)).await
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(&self.current);
self.current.compute(&ctx, opt).await
}
Statement::Insert(_) => {
self.current.compute(ctx, opt, txn, Some(&self.current)).await
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(&self.current);
self.current.compute(&ctx, opt).await
}
_ => Err(Error::Ignore),
},
@ -57,14 +81,16 @@ impl<'a> Document<'a> {
if self.id.is_some() {
// Should we run permissions checks?
if opt.perms && opt.auth.perms() {
// Clone transaction
let txn = ctx.clone_transaction()?;
// Loop through all field statements
for fd in self.fd(opt, txn).await?.iter() {
for fd in self.fd(opt, &txn).await?.iter() {
// Loop over each field in document
for k in out.each(&fd.name).iter() {
// Process the field permissions
match &fd.permissions.select {
Permission::Full => (),
Permission::None => out.del(ctx, opt, txn, k).await?,
Permission::None => out.del(ctx, opt, k).await?,
Permission::Specific(e) => {
// Disable permissions
let opt = &opt.perms(false);
@ -73,13 +99,10 @@ impl<'a> Document<'a> {
// Configure the context
let mut ctx = Context::new(ctx);
ctx.add_value("value", &val);
ctx.add_cursor_doc(&self.current);
// Process the PERMISSION clause
if !e
.compute(&ctx, opt, txn, Some(&self.current))
.await?
.is_truthy()
{
out.del(&ctx, opt, txn, k).await?
if !e.compute(&ctx, opt).await?.is_truthy() {
out.del(&ctx, opt, k).await?
}
}
}
@ -88,7 +111,7 @@ impl<'a> Document<'a> {
}
}
// Remove metadata fields on output
out.del(ctx, opt, txn, &*META).await?;
out.del(ctx, opt, &*META).await?;
// Output result
Ok(out)
}

79
lib/src/doc/purge.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
use crate::sql::dir::Dir;
@ -18,15 +17,16 @@ impl<'a> Document<'a> {
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_stm: &Statement<'_>,
) -> Result<(), Error> {
// Check if forced
if !opt.force && !self.changed() {
return Ok(());
}
// Clone transaction
let txn = ctx.clone_transaction()?;
// Check if the table is a view
if self.tb(opt, txn).await?.drop {
if self.tb(opt, &txn).await?.drop {
return Ok(());
}
// Clone transaction
@ -34,42 +34,43 @@ impl<'a> Document<'a> {
// Claim transaction
let mut run = run.lock().await;
// Get the record id
let rid = self.id.as_ref().unwrap();
// Purge the record data
let key = crate::key::thing::new(opt.ns(), opt.db(), &rid.tb, &rid.id);
run.del(key).await?;
// Purge the record edges
match (self.initial.pick(&*EDGE), self.initial.pick(&*IN), self.initial.pick(&*OUT)) {
(Value::Bool(true), Value::Thing(ref l), Value::Thing(ref r)) => {
// Get temporary edge references
let (ref o, ref i) = (Dir::Out, Dir::In);
// Purge the left pointer edge
let key = crate::key::graph::new(opt.ns(), opt.db(), &l.tb, &l.id, o, rid);
run.del(key).await?;
// Purge the left inner edge
let key = crate::key::graph::new(opt.ns(), opt.db(), &rid.tb, &rid.id, i, l);
run.del(key).await?;
// Purge the right inner edge
let key = crate::key::graph::new(opt.ns(), opt.db(), &rid.tb, &rid.id, o, r);
run.del(key).await?;
// Purge the right pointer edge
let key = crate::key::graph::new(opt.ns(), opt.db(), &r.tb, &r.id, i, rid);
run.del(key).await?;
}
_ => {
// Release the transaction
drop(run);
// Setup the delete statement
let stm = DeleteStatement {
what: Values(vec![Value::from(Edges {
dir: Dir::Both,
from: rid.clone(),
what: Tables::default(),
})]),
..DeleteStatement::default()
};
// Execute the delete statement
stm.compute(ctx, opt, txn, None).await?;
if let Some(rid) = self.id {
// Purge the record data
let key = crate::key::thing::new(opt.ns(), opt.db(), &rid.tb, &rid.id);
run.del(key).await?;
// Purge the record edges
match (self.initial.pick(&*EDGE), self.initial.pick(&*IN), self.initial.pick(&*OUT)) {
(Value::Bool(true), Value::Thing(ref l), Value::Thing(ref r)) => {
// Get temporary edge references
let (ref o, ref i) = (Dir::Out, Dir::In);
// Purge the left pointer edge
let key = crate::key::graph::new(opt.ns(), opt.db(), &l.tb, &l.id, o, rid);
run.del(key).await?;
// Purge the left inner edge
let key = crate::key::graph::new(opt.ns(), opt.db(), &rid.tb, &rid.id, i, l);
run.del(key).await?;
// Purge the right inner edge
let key = crate::key::graph::new(opt.ns(), opt.db(), &rid.tb, &rid.id, o, r);
run.del(key).await?;
// Purge the right pointer edge
let key = crate::key::graph::new(opt.ns(), opt.db(), &r.tb, &r.id, i, rid);
run.del(key).await?;
}
_ => {
// Release the transaction
drop(run);
// Setup the delete statement
let stm = DeleteStatement {
what: Values(vec![Value::from(Edges {
dir: Dir::Both,
from: rid.clone(),
what: Tables::default(),
})]),
..DeleteStatement::default()
};
// Execute the delete statement
stm.compute(ctx, opt).await?;
}
}
}
// Carry on

28
lib/src/doc/relate.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
use crate::sql::value::Value;
@ -11,34 +10,33 @@ impl<'a> Document<'a> {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<Value, Error> {
// Check if allowed
self.allow(ctx, opt, txn, stm).await?;
self.allow(ctx, opt, stm).await?;
// Alter record data
self.alter(ctx, opt, txn, stm).await?;
self.alter(ctx, opt, stm).await?;
// Merge fields data
self.field(ctx, opt, txn, stm).await?;
self.field(ctx, opt, stm).await?;
// Reset fields data
self.reset(ctx, opt, txn, stm).await?;
self.reset(ctx, opt, stm).await?;
// Clean fields data
self.clean(ctx, opt, txn, stm).await?;
self.clean(ctx, opt, stm).await?;
// Check if allowed
self.allow(ctx, opt, txn, stm).await?;
self.allow(ctx, opt, stm).await?;
// Store record edges
self.edges(ctx, opt, txn, stm).await?;
self.edges(ctx, opt, stm).await?;
// Store index data
self.index(ctx, opt, txn, stm).await?;
self.index(ctx, opt, stm).await?;
// Store record data
self.store(ctx, opt, txn, stm).await?;
self.store(ctx, opt, stm).await?;
// Run table queries
self.table(ctx, opt, txn, stm).await?;
self.table(ctx, opt, stm).await?;
// Run lives queries
self.lives(ctx, opt, txn, stm).await?;
self.lives(ctx, opt, stm).await?;
// Run event queries
self.event(ctx, opt, txn, stm).await?;
self.event(ctx, opt, stm).await?;
// Yield document
self.pluck(ctx, opt, txn, stm).await
self.pluck(ctx, opt, stm).await
}
}

2
lib/src/doc/reset.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
use crate::sql::paths::EDGE;
@ -14,7 +13,6 @@ impl<'a> Document<'a> {
&mut self,
_ctx: &Context<'_>,
_opt: &Options,
_txn: &Transaction,
_stm: &Statement<'_>,
) -> Result<(), Error> {
// Get the record id

10
lib/src/doc/select.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
use crate::sql::value::Value;
@ -11,16 +10,15 @@ impl<'a> Document<'a> {
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<Value, Error> {
// Check if record exists
self.empty(ctx, opt, txn, stm).await?;
self.empty(ctx, opt, stm).await?;
// Check where clause
self.check(ctx, opt, txn, stm).await?;
self.check(ctx, opt, stm).await?;
// Check if allowed
self.allow(ctx, opt, txn, stm).await?;
self.allow(ctx, opt, stm).await?;
// Yield document
self.pluck(ctx, opt, txn, stm).await
self.pluck(ctx, opt, stm).await
}
}

12
lib/src/doc/store.rs
View file

@ -1,30 +1,28 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
impl<'a> Document<'a> {
pub async fn store(
&self,
_ctx: &Context<'_>,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_stm: &Statement<'_>,
) -> Result<(), Error> {
// Check if forced
if !opt.force && !self.changed() {
return Ok(());
}
// Clone transaction
let txn = ctx.clone_transaction()?;
// Check if the table is a view
if self.tb(opt, txn).await?.drop {
if self.tb(opt, &txn).await?.drop {
return Ok(());
}
// Clone transaction
let run = txn.clone();
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Get the record id
let rid = self.id.as_ref().unwrap();
// Store the record data

99
lib/src/doc/table.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
use crate::sql::data::Data;
@ -34,7 +33,6 @@ impl<'a> Document<'a> {
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<(), Error> {
// Check events
@ -57,41 +55,43 @@ impl<'a> Document<'a> {
} else {
Action::Update
};
// Clone transaction
let txn = ctx.clone_transaction()?;
// Loop through all foreign table statements
for ft in self.ft(opt, txn).await?.iter() {
for ft in self.ft(opt, &txn).await?.iter() {
// Get the table definition
let tb = ft.view.as_ref().unwrap();
// Check if there is a GROUP BY clause
match &tb.group {
// There is a GROUP BY clause specified
Some(group) => {
let mut initial_ctx = Context::new(ctx);
initial_ctx.add_cursor_doc(&self.initial);
// Set the previous record id
let old = Thing {
tb: ft.name.to_raw(),
id: try_join_all(
group.iter().map(|v| v.compute(ctx, opt, txn, Some(&self.initial))),
)
.await?
.into_iter()
.collect::<Vec<_>>()
.into(),
id: try_join_all(group.iter().map(|v| v.compute(&initial_ctx, opt)))
.await?
.into_iter()
.collect::<Vec<_>>()
.into(),
};
let mut current_ctx = Context::new(ctx);
current_ctx.add_cursor_doc(&self.current);
// Set the current record id
let rid = Thing {
tb: ft.name.to_raw(),
id: try_join_all(
group.iter().map(|v| v.compute(ctx, opt, txn, Some(&self.current))),
)
.await?
.into_iter()
.collect::<Vec<_>>()
.into(),
id: try_join_all(group.iter().map(|v| v.compute(&current_ctx, opt)))
.await?
.into_iter()
.collect::<Vec<_>>()
.into(),
};
// Check if a WHERE clause is specified
match &tb.cond {
// There is a WHERE clause specified
Some(cond) => {
match cond.compute(ctx, opt, txn, Some(&self.current)).await? {
match cond.compute(&current_ctx, opt).await? {
v if v.is_truthy() => {
if !opt.force && act != Action::Create {
// Delete the old value
@ -99,13 +99,11 @@ impl<'a> Document<'a> {
// Modify the value in the table
let stm = UpdateStatement {
what: Values(vec![Value::from(old)]),
data: Some(
self.data(ctx, opt, txn, act, &tb.expr).await?,
),
data: Some(self.data(ctx, opt, act, &tb.expr).await?),
..UpdateStatement::default()
};
// Execute the statement
stm.compute(ctx, opt, txn, None).await?;
stm.compute(ctx, opt).await?;
}
if act != Action::Delete {
// Update the new value
@ -113,13 +111,11 @@ impl<'a> Document<'a> {
// Modify the value in the table
let stm = UpdateStatement {
what: Values(vec![Value::from(rid)]),
data: Some(
self.data(ctx, opt, txn, act, &tb.expr).await?,
),
data: Some(self.data(ctx, opt, act, &tb.expr).await?),
..UpdateStatement::default()
};
// Execute the statement
stm.compute(ctx, opt, txn, None).await?;
stm.compute(ctx, opt).await?;
}
}
_ => {
@ -129,13 +125,11 @@ impl<'a> Document<'a> {
// Modify the value in the table
let stm = UpdateStatement {
what: Values(vec![Value::from(old)]),
data: Some(
self.data(ctx, opt, txn, act, &tb.expr).await?,
),
data: Some(self.data(ctx, opt, act, &tb.expr).await?),
..UpdateStatement::default()
};
// Execute the statement
stm.compute(ctx, opt, txn, None).await?;
stm.compute(ctx, opt).await?;
}
}
}
@ -148,11 +142,11 @@ impl<'a> Document<'a> {
// Modify the value in the table
let stm = UpdateStatement {
what: Values(vec![Value::from(old)]),
data: Some(self.data(ctx, opt, txn, act, &tb.expr).await?),
data: Some(self.data(ctx, opt, act, &tb.expr).await?),
..UpdateStatement::default()
};
// Execute the statement
stm.compute(ctx, opt, txn, None).await?;
stm.compute(ctx, opt).await?;
}
if act != Action::Delete {
// Update the new value
@ -160,11 +154,11 @@ impl<'a> Document<'a> {
// Modify the value in the table
let stm = UpdateStatement {
what: Values(vec![Value::from(rid)]),
data: Some(self.data(ctx, opt, txn, act, &tb.expr).await?),
data: Some(self.data(ctx, opt, act, &tb.expr).await?),
..UpdateStatement::default()
};
// Execute the statement
stm.compute(ctx, opt, txn, None).await?;
stm.compute(ctx, opt).await?;
}
}
}
@ -177,12 +171,13 @@ impl<'a> Document<'a> {
id: rid.id.clone(),
};
// Use the current record data
let doc = Some(self.current.as_ref());
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(&self.current);
// Check if a WHERE clause is specified
match &tb.cond {
// There is a WHERE clause specified
Some(cond) => {
match cond.compute(ctx, opt, txn, doc).await? {
match cond.compute(&ctx, opt).await? {
v if v.is_truthy() => {
// Define the statement
let stm = match act {
@ -195,13 +190,13 @@ impl<'a> Document<'a> {
_ => Query::Update(UpdateStatement {
what: Values(vec![Value::from(rid)]),
data: Some(Data::ReplaceExpression(
tb.expr.compute(ctx, opt, txn, doc, false).await?,
tb.expr.compute(&ctx, opt, false).await?,
)),
..UpdateStatement::default()
}),
};
// Execute the statement
stm.compute(ctx, opt, txn, None).await?;
stm.compute(&ctx, opt).await?;
}
_ => {
// Delete the value in the table
@ -210,7 +205,7 @@ impl<'a> Document<'a> {
..DeleteStatement::default()
};
// Execute the statement
stm.compute(ctx, opt, txn, None).await?;
stm.compute(&ctx, opt).await?;
}
}
}
@ -227,13 +222,13 @@ impl<'a> Document<'a> {
_ => Query::Update(UpdateStatement {
what: Values(vec![Value::from(rid)]),
data: Some(Data::ReplaceExpression(
tb.expr.compute(ctx, opt, txn, doc, false).await?,
tb.expr.compute(&ctx, opt, false).await?,
)),
..UpdateStatement::default()
}),
};
// Execute the statement
stm.compute(ctx, opt, txn, None).await?;
stm.compute(&ctx, opt).await?;
}
}
}
@ -247,16 +242,16 @@ impl<'a> Document<'a> {
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
act: Action,
exp: &Fields,
) -> Result<Data, Error> {
//
let mut ops: Ops = vec![];
//
let doc = match act {
Action::Delete => Some(self.initial.as_ref()),
Action::Update => Some(self.current.as_ref()),
// Create a new context with the initial or the current doc
let mut ctx = Context::new(ctx);
match act {
Action::Delete => ctx.add_cursor_doc(self.initial.as_ref()),
Action::Update => ctx.add_cursor_doc(self.current.as_ref()),
_ => unreachable!(),
};
//
@ -271,29 +266,29 @@ impl<'a> Document<'a> {
match expr {
Value::Function(f) if f.is_rolling() => match f.name() {
"count" => {
let val = f.compute(ctx, opt, txn, doc).await?;
let val = f.compute(&ctx, opt).await?;
self.chg(&mut ops, &act, idiom, val);
}
"math::sum" => {
let val = f.args()[0].compute(ctx, opt, txn, doc).await?;
let val = f.args()[0].compute(&ctx, opt).await?;
self.chg(&mut ops, &act, idiom, val);
}
"math::min" => {
let val = f.args()[0].compute(ctx, opt, txn, doc).await?;
let val = f.args()[0].compute(&ctx, opt).await?;
self.min(&mut ops, &act, idiom, val);
}
"math::max" => {
let val = f.args()[0].compute(ctx, opt, txn, doc).await?;
let val = f.args()[0].compute(&ctx, opt).await?;
self.max(&mut ops, &act, idiom, val);
}
"math::mean" => {
let val = f.args()[0].compute(ctx, opt, txn, doc).await?;
let val = f.args()[0].compute(&ctx, opt).await?;
self.mean(&mut ops, &act, idiom, val);
}
_ => unreachable!(),
},
_ => {
let val = expr.compute(ctx, opt, txn, doc).await?;
let val = expr.compute(&ctx, opt).await?;
self.set(&mut ops, idiom, val);
}
}

28
lib/src/doc/update.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::doc::Document;
use crate::err::Error;
use crate::sql::value::Value;
@ -11,34 +10,33 @@ impl<'a> Document<'a> {
&mut self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
stm: &Statement<'_>,
) -> Result<Value, Error> {
// Check where clause
self.check(ctx, opt, txn, stm).await?;
self.check(ctx, opt, stm).await?;
// Check if allowed
self.allow(ctx, opt, txn, stm).await?;
self.allow(ctx, opt, stm).await?;
// Alter record data
self.alter(ctx, opt, txn, stm).await?;
self.alter(ctx, opt, stm).await?;
// Merge fields data
self.field(ctx, opt, txn, stm).await?;
self.field(ctx, opt, stm).await?;
// Reset fields data
self.reset(ctx, opt, txn, stm).await?;
self.reset(ctx, opt, stm).await?;
// Clean fields data
self.clean(ctx, opt, txn, stm).await?;
self.clean(ctx, opt, stm).await?;
// Check if allowed
self.allow(ctx, opt, txn, stm).await?;
self.allow(ctx, opt, stm).await?;
// Store index data
self.index(ctx, opt, txn, stm).await?;
self.index(ctx, opt, stm).await?;
// Store record data
self.store(ctx, opt, txn, stm).await?;
self.store(ctx, opt, stm).await?;
// Run table queries
self.table(ctx, opt, txn, stm).await?;
self.table(ctx, opt, stm).await?;
// Run lives queries
self.lives(ctx, opt, txn, stm).await?;
self.lives(ctx, opt, stm).await?;
// Run event queries
self.event(ctx, opt, txn, stm).await?;
self.event(ctx, opt, stm).await?;
// Yield document
self.pluck(ctx, opt, txn, stm).await
self.pluck(ctx, opt, stm).await
}
}

22
lib/src/err/mod.rs
View file

@ -61,6 +61,10 @@ pub enum Error {
#[error("Transaction is too large")]
TxTooLarge,
/// The context does have any transaction
#[error("No transaction")]
NoTx,
/// No namespace has been selected
#[error("Specify a namespace to use")]
NsEmpty,
@ -267,6 +271,12 @@ pub enum Error {
value: String,
},
/// The requested analyzer does not exist
#[error("The index '{value}' does not exist")]
IxNotFound {
value: String,
},
/// Unable to perform the realtime query
#[error("Unable to perform the realtime query")]
RealtimeDisabled,
@ -432,10 +442,16 @@ pub enum Error {
#[error("Key decoding error: {0}")]
Decode(#[from] DecodeError),
/// Represents an error when decoding a key-value entry
/// The index has been found to be inconsistent
#[error("Index is corrupted")]
CorruptedIndex,
/// The query planner did not find an index able to support the match @@ operator on a given expression
#[error("There was no suitable full-text index supporting the expression '{value}'")]
NoIndexFoundForMatch {
value: String,
},
/// Represents an error when analyzing a value
#[error("A string can't be analyzed: {0}")]
AnalyzerError(String),
@ -457,6 +473,10 @@ pub enum Error {
FeatureNotYetImplemented {
feature: &'static str,
},
#[doc(hidden)]
#[error("Bypass the query planner")]
BypassQueryPlanner,
}
impl From<Error> for String {

14
lib/src/fnc/operate.rs
View file

@ -1,3 +1,4 @@
use crate::ctx::Context;
use crate::err::Error;
use crate::sql::value::TryAdd;
use crate::sql::value::TryDiv;
@ -5,6 +6,7 @@ use crate::sql::value::TryMul;
use crate::sql::value::TryPow;
use crate::sql::value::TrySub;
use crate::sql::value::Value;
use crate::sql::Expression;
pub fn or(a: Value, b: Value) -> Result<Value, Error> {
Ok(match a.is_truthy() {
@ -154,6 +156,18 @@ pub fn intersects(a: &Value, b: &Value) -> Result<Value, Error> {
Ok(a.intersects(b).into())
}
pub(crate) async fn matches(ctx: &Context<'_>, e: &Expression) -> Result<Value, Error> {
if let Some(thg) = ctx.thing() {
if let Some(exe) = ctx.get_query_executor(&thg.tb) {
// Clone transaction
let txn = ctx.clone_transaction()?;
// Check the matches
return exe.matches(&txn, thg, e).await;
}
}
Ok(Value::Bool(false))
}
#[cfg(test)]
mod tests {

5
lib/src/fnc/script/main.rs
View file

@ -5,7 +5,6 @@ use super::modules::loader;
use super::modules::resolver;
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::value::Value;
use js::async_with;
@ -19,8 +18,6 @@ use js::Module;
pub async fn run(
ctx: &Context<'_>,
_opt: &Options,
_txn: &Transaction,
doc: Option<&Value>,
src: &str,
arg: Vec<Value>,
) -> Result<Value, Error> {
@ -28,6 +25,8 @@ pub async fn run(
if ctx.is_done() {
return Ok(Value::None);
}
// Get the optional doc
let doc = ctx.doc();
// Create an JavaScript context
let run = js::AsyncRuntime::new().unwrap();
// Explicitly set max stack size to 256 KiB

14
lib/src/idx/bkeys.rs
View file

@ -11,7 +11,7 @@ use std::io;
pub(super) trait BKeys: Display + Sized {
fn with_key_val(key: Key, payload: Payload) -> Result<Self, Error>;
fn len(&self) -> usize;
fn len(&self) -> u32;
fn get(&self, key: &Key) -> Option<Payload>;
// It is okay to return a owned Vec rather than an iterator,
// because BKeys are intended to be stored as Node in the BTree.
@ -48,7 +48,7 @@ pub(super) struct FstKeys {
map: Map<Vec<u8>>,
additions: Trie<Key, Payload>,
deletions: Trie<Key, bool>,
len: usize,
len: u32,
}
impl BKeys for FstKeys {
@ -58,7 +58,7 @@ impl BKeys for FstKeys {
Ok(Self::try_from(builder)?)
}
fn len(&self) -> usize {
fn len(&self) -> u32 {
self.len
}
@ -255,7 +255,7 @@ impl TryFrom<Vec<u8>> for FstKeys {
type Error = fst::Error;
fn try_from(bytes: Vec<u8>) -> Result<Self, Self::Error> {
let map = Map::new(bytes)?;
let len = map.len();
let len = map.len() as u32;
Ok(Self {
map,
len,
@ -371,8 +371,8 @@ impl BKeys for TrieKeys {
Ok(trie_keys)
}
fn len(&self) -> usize {
self.keys.len()
fn len(&self) -> u32 {
self.keys.len() as u32
}
fn get(&self, key: &Key) -> Option<Payload> {
@ -563,7 +563,7 @@ mod tests {
keys.insert(key.clone(), i);
keys.compile();
assert_eq!(keys.get(&key), Some(i));
assert_eq!(keys.len(), term_set.len());
assert_eq!(keys.len() as usize, term_set.len());
i += 1;
}
}

400
lib/src/idx/btree.rs
View file

@ -2,13 +2,12 @@ use crate::err::Error;
use crate::idx::bkeys::BKeys;
use crate::idx::SerdeState;
use crate::kvs::{Key, Transaction};
use crate::sql::{Object, Value};
use async_trait::async_trait;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::collections::VecDeque;
use std::fmt::Debug;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
pub(crate) type NodeId = u64;
pub(super) type Payload = u64;
@ -41,13 +40,13 @@ where
{
keys: K,
state: State,
full_size: usize,
full_size: u32,
updated: bool,
}
#[derive(Clone, Serialize, Deserialize)]
pub(super) struct State {
minimum_degree: usize,
minimum_degree: u32,
root: Option<NodeId>,
next_node_id: NodeId,
}
@ -55,7 +54,7 @@ pub(super) struct State {
impl SerdeState for State {}
impl State {
pub(super) fn new(minimum_degree: usize) -> Self {
pub(super) fn new(minimum_degree: u32) -> Self {
assert!(minimum_degree >= 2, "Minimum degree should be >= 2");
Self {
minimum_degree,
@ -67,10 +66,21 @@ impl State {
#[derive(Debug, Default, PartialEq)]
pub(super) struct Statistics {
pub(super) keys_count: usize,
pub(super) max_depth: usize,
pub(super) nodes_count: usize,
pub(super) total_size: usize,
pub(super) keys_count: u64,
pub(super) max_depth: u32,
pub(super) nodes_count: u32,
pub(super) total_size: u64,
}
impl From<Statistics> for Value {
fn from(stats: Statistics) -> Self {
let mut res = Object::default();
res.insert("keys_count".to_owned(), Value::from(stats.keys_count));
res.insert("max_depth".to_owned(), Value::from(stats.max_depth));
res.insert("nodes_count".to_owned(), Value::from(stats.nodes_count));
res.insert("total_size".to_owned(), Value::from(stats.total_size));
Value::from(res)
}
}
#[derive(Serialize, Deserialize)]
@ -86,21 +96,21 @@ impl<'a, BK> Node<BK>
where
BK: BKeys + Serialize + DeserializeOwned + 'a,
{
async fn read(tx: &mut Transaction, key: Key) -> Result<(Self, usize), Error> {
async fn read(tx: &mut Transaction, key: Key) -> Result<(Self, u32), Error> {
if let Some(val) = tx.get(key).await? {
let size = val.len();
let size = val.len() as u32;
Ok((Node::try_from_val(val)?, size))
} else {
Err(Error::CorruptedIndex)
}
}
async fn write(&mut self, tx: &mut Transaction, key: Key) -> Result<usize, Error> {
async fn write(&mut self, tx: &mut Transaction, key: Key) -> Result<u32, Error> {
self.keys_mut().compile();
let val = self.try_to_val()?;
let size = val.len();
tx.set(key, val).await?;
Ok(size)
Ok(size as u32)
}
fn keys(&self) -> &BK {
@ -140,20 +150,6 @@ where
}
}
}
fn debug<F>(&self, to_string: F) -> Result<(), Error>
where
F: Fn(Key) -> Result<String, Error>,
{
match self {
Node::Internal(keys, children) => {
keys.debug(to_string)?;
debug!("Children{:?}", children);
Ok(())
}
Node::Leaf(keys) => keys.debug(to_string),
}
}
}
impl<BK> SerdeState for Node<BK> where BK: BKeys + Serialize + DeserializeOwned {}
@ -203,10 +199,6 @@ where
Ok(None)
}
pub(super) fn search_by_prefix(&self, prefix_key: Key) -> BTreeIterator<K> {
BTreeIterator::new(self.keys.clone(), prefix_key, self.state.root)
}
pub(super) async fn insert<BK>(
&mut self,
tx: &mut Transaction,
@ -218,7 +210,7 @@ where
{
if let Some(root_id) = self.state.root {
let root = self.keys.load_node::<BK>(tx, root_id).await?;
if root.node.keys().len() == self.full_size {
if root.node.keys().len() as u32 == self.full_size {
let new_root_id = self.new_node_id();
let new_root =
self.new_node(new_root_id, Node::Internal(BK::default(), vec![root_id]));
@ -264,7 +256,7 @@ where
}
let child_idx = keys.get_child_idx(&key);
let child = self.keys.load_node::<BK>(tx, children[child_idx]).await?;
let next = if child.node.keys().len() == self.full_size {
let next = if child.node.keys().len() as u32 == self.full_size {
let split_result =
self.split_child::<BK>(tx, node, child_idx, child).await?;
if key.gt(&split_result.median_key) {
@ -439,7 +431,7 @@ where
let left_idx = keys.get_child_idx(&key_to_delete);
let left_id = children[left_idx];
let mut left_node = self.keys.load_node::<BK>(tx, left_id).await?;
if left_node.node.keys().len() >= self.state.minimum_degree {
if left_node.node.keys().len() as u32 >= self.state.minimum_degree {
// CLRS: 2a -> left_node is named `y` in the book
if let Some((key_prim, payload_prim)) = left_node.node.keys().get_last_key() {
keys.remove(&key_to_delete);
@ -653,35 +645,6 @@ where
Err(Error::CorruptedIndex)
}
/// This is for debugging
async fn inspect_nodes<BK, F>(
&self,
tx: &mut Transaction,
inspect_func: F,
) -> Result<usize, Error>
where
F: Fn(usize, usize, NodeId, StoredNode<BK>),
BK: BKeys + Serialize + DeserializeOwned,
{
let mut node_queue = VecDeque::new();
if let Some(node_id) = self.state.root {
node_queue.push_front((node_id, 1));
}
let mut count = 0;
while let Some((node_id, depth)) = node_queue.pop_front() {
let stored_node = self.keys.load_node::<BK>(tx, node_id).await?;
if let Node::Internal(_, children) = &stored_node.node {
let depth = depth + 1;
for child_id in children {
node_queue.push_back((*child_id, depth));
}
}
inspect_func(count, depth, node_id, stored_node);
count += 1;
}
Ok(count)
}
pub(super) async fn statistics<BK>(&self, tx: &mut Transaction) -> Result<Statistics, Error>
where
BK: BKeys + Serialize + DeserializeOwned,
@ -693,12 +656,12 @@ where
}
while let Some((node_id, depth)) = node_queue.pop_front() {
let stored = self.keys.load_node::<BK>(tx, node_id).await?;
stats.keys_count += stored.node.keys().len();
stats.keys_count += stored.node.keys().len() as u64;
if depth > stats.max_depth {
stats.max_depth = depth;
}
stats.nodes_count += 1;
stats.total_size += stored.size;
stats.total_size += stored.size as u64;
if let Node::Internal(_, children) = stored.node {
let depth = depth + 1;
for child_id in &children {
@ -730,94 +693,6 @@ where
}
}
struct CurrentNode {
keys: VecDeque<(Key, Payload)>,
matches_found: bool,
level_matches_found: Arc<AtomicBool>,
}
pub(super) struct BTreeIterator<K>
where
K: KeyProvider,
{
key_provider: K,
prefix_key: Key,
node_queue: VecDeque<(NodeId, Arc<AtomicBool>)>,
current_node: Option<CurrentNode>,
}
impl<K> BTreeIterator<K>
where
K: KeyProvider + Sync,
{
fn new(key_provider: K, prefix_key: Key, start_node: Option<NodeId>) -> Self {
let mut node_queue = VecDeque::new();
if let Some(node_id) = start_node {
node_queue.push_front((node_id, Arc::new(AtomicBool::new(false))))
}
Self {
key_provider,
prefix_key,
node_queue,
current_node: None,
}
}
fn set_current_node<BK>(&mut self, node: Node<BK>, level_matches_found: Arc<AtomicBool>)
where
BK: BKeys + Serialize + DeserializeOwned,
{
if let Node::Internal(keys, children) = &node {
let same_level_matches_found = Arc::new(AtomicBool::new(false));
let child_idx = keys.get_child_idx(&self.prefix_key);
for i in child_idx..children.len() {
self.node_queue.push_front((children[i], same_level_matches_found.clone()));
}
}
let keys = node.keys().collect_with_prefix(&self.prefix_key);
let matches_found = !keys.is_empty();
if matches_found {
level_matches_found.fetch_and(true, Ordering::Relaxed);
}
self.current_node = Some(CurrentNode {
keys,
matches_found,
level_matches_found,
});
}
pub(super) async fn next<BK>(
&mut self,
tx: &mut Transaction,
) -> Result<Option<(Key, Payload)>, Error>
where
BK: BKeys + Serialize + DeserializeOwned,
{
loop {
if let Some(current) = &mut self.current_node {
if let Some((key, payload)) = current.keys.pop_front() {
return Ok(Some((key, payload)));
} else {
if !current.matches_found && current.level_matches_found.load(Ordering::Relaxed)
{
// If we have found matches in previous (lower) nodes,
// but we don't find matches anymore, there is no chance we can find new matches
// in upper child nodes, therefore we can stop the traversal.
break;
}
self.current_node = None;
}
} else if let Some((node_id, level_matches_found)) = self.node_queue.pop_front() {
let st = self.key_provider.load_node::<BK>(tx, node_id).await?;
self.set_current_node(st.node, level_matches_found);
} else {
break;
}
}
Ok(None)
}
}
pub(super) struct StoredNode<BK>
where
BK: BKeys,
@ -825,7 +700,7 @@ where
node: Node<BK>,
id: NodeId,
key: Key,
size: usize,
size: u32,
}
impl<BK> StoredNode<BK>
@ -840,17 +715,18 @@ where
#[cfg(test)]
mod tests {
use crate::err::Error;
use crate::idx::bkeys::{BKeys, FstKeys, TrieKeys};
use crate::idx::btree::{
BTree, BTreeIterator, KeyProvider, Node, NodeId, Payload, State, Statistics,
BTree, KeyProvider, Node, NodeId, Payload, State, Statistics, StoredNode,
};
use crate::idx::SerdeState;
use crate::kvs::{Datastore, Key, Transaction};
use rand::prelude::{SliceRandom, ThreadRng};
use rand::prelude::SliceRandom;
use rand::thread_rng;
use serde::de::DeserializeOwned;
use serde::Serialize;
use std::collections::HashMap;
use std::collections::{HashMap, VecDeque};
use test_log::test;
#[derive(Clone)]
@ -1025,7 +901,7 @@ mod tests {
"nothing", "the", "other", "animals", "sat", "there", "watching",
];
async fn test_btree_read_world_insertions<BK>(default_minimum_degree: usize) -> Statistics
async fn test_btree_read_world_insertions<BK>(default_minimum_degree: u32) -> Statistics
where
BK: BKeys + Serialize + DeserializeOwned + Default,
{
@ -1097,159 +973,6 @@ mod tests {
);
}
async fn test_btree_search_by_prefix(
ds: &Datastore,
minimum_degree: usize,
shuffle: bool,
mut samples: Vec<(&str, Payload)>,
) -> (BTree<TestKeyProvider>, Statistics) {
let mut tx = ds.transaction(true, false).await.unwrap();
let mut t = BTree::new(TestKeyProvider {}, State::new(minimum_degree));
let samples_len = samples.len();
if shuffle {
samples.shuffle(&mut ThreadRng::default());
}
for (key, payload) in samples {
t.insert::<TrieKeys>(&mut tx, key.into(), payload).await.unwrap();
}
tx.commit().await.unwrap();
let mut tx = ds.transaction(false, false).await.unwrap();
let s = t.statistics::<TrieKeys>(&mut tx).await.unwrap();
assert_eq!(s.keys_count, samples_len);
(t, s)
}
async fn check_results(
mut i: BTreeIterator<TestKeyProvider>,
tx: &mut Transaction,
e: Vec<(Key, Payload)>,
) {
let mut map = HashMap::new();
while let Some((k, p)) = i.next::<TrieKeys>(tx).await.unwrap() {
map.insert(k, p);
}
assert_eq!(map.len(), e.len());
for (k, p) in e {
assert_eq!(map.get(&k), Some(&p));
}
}
#[test(tokio::test)]
async fn test_btree_trie_keys_search_by_prefix() {
for _ in 0..50 {
let samples = vec![
("aaaa", 0),
("bb1", 21),
("bb2", 22),
("bb3", 23),
("bb4", 24),
("dddd", 0),
("eeee", 0),
("ffff", 0),
("gggg", 0),
("hhhh", 0),
];
let ds = Datastore::new("memory").await.unwrap();
let (t, s) = test_btree_search_by_prefix(&ds, 3, true, samples).await;
// For this test to be relevant, we expect the BTree to match the following properties:
assert_eq!(s.max_depth, 2, "Tree depth should be 2");
assert!(
s.nodes_count > 2 && s.nodes_count < 5,
"The number of node should be between 3 and 4 inclusive"
);
let mut tx = ds.transaction(false, false).await.unwrap();
// We should find all the keys prefixed with "bb"
let i = t.search_by_prefix("bb".into());
check_results(
i,
&mut tx,
vec![
("bb1".into(), 21),
("bb2".into(), 22),
("bb3".into(), 23),
("bb4".into(), 24),
],
)
.await;
}
}
#[test(tokio::test)]
async fn test_btree_trie_keys_real_world_search_by_prefix() {
// We do multiples tests to run the test over many different possible forms of Tree.
// The samples are shuffled, therefore the insertion order is different on each test,
// ending up in slightly different variants of the BTrees.
for _ in 0..50 {
// This samples simulate postings. Pair of terms and document ids.
let samples = vec![
("the-1", 0),
("quick-1", 0),
("brown-1", 0),
("fox-1", 0),
("jumped-1", 0),
("over-1", 0),
("lazy-1", 0),
("dog-1", 0),
("the-2", 0),
("fast-2", 0),
("fox-2", 0),
("jumped-2", 0),
("over-2", 0),
("lazy-2", 0),
("dog-2", 0),
("the-3", 0),
("dog-3", 0),
("sat-3", 0),
("there-3", 0),
("and-3", 0),
("did-3", 0),
("nothing-3", 0),
("the-4", 0),
("other-4", 0),
("animals-4", 0),
("sat-4", 0),
("there-4", 0),
("watching-4", 0),
];
let ds = Datastore::new("memory").await.unwrap();
let (t, s) = test_btree_search_by_prefix(&ds, 7, true, samples).await;
// For this test to be relevant, we expect the BTree to match the following properties:
assert_eq!(s.max_depth, 2, "Tree depth should be 2");
assert!(
s.nodes_count > 2 && s.nodes_count < 5,
"The number of node should be > 2 and < 5"
);
let mut tx = ds.transaction(false, false).await.unwrap();
for (prefix, count) in vec![
("the", 6),
("there", 2),
("dog", 3),
("jumped", 2),
("lazy", 2),
("fox", 2),
("over", 2),
("sat", 2),
("other", 1),
("nothing", 1),
("animals", 1),
("watching", 1),
] {
let mut i = t.search_by_prefix(prefix.into());
for _ in 0..count {
assert!(i.next::<TrieKeys>(&mut tx).await.unwrap().is_some());
}
assert_eq!(i.next::<TrieKeys>(&mut tx).await.unwrap(), None);
}
}
}
// This is the examples from the chapter B-Trees in CLRS:
// https://en.wikipedia.org/wiki/Introduction_to_Algorithms
const CLRS_EXAMPLE: [(&str, Payload); 23] = [
@ -1554,7 +1277,7 @@ mod tests {
where
BK: BKeys + Serialize + DeserializeOwned,
{
assert_eq!(keys.len(), expected_keys.len(), "The number of keys does not match");
assert_eq!(keys.len() as usize, expected_keys.len(), "The number of keys does not match");
for (key, payload) in expected_keys {
assert_eq!(
keys.get(&key.into()),
@ -1564,4 +1287,57 @@ mod tests {
);
}
}
impl<K> BTree<K>
where
K: KeyProvider + Clone + Sync,
{
/// This is for debugging
async fn inspect_nodes<BK, F>(
&self,
tx: &mut Transaction,
inspect_func: F,
) -> Result<usize, Error>
where
F: Fn(usize, usize, NodeId, StoredNode<BK>),
BK: BKeys + Serialize + DeserializeOwned,
{
let mut node_queue = VecDeque::new();
if let Some(node_id) = self.state.root {
node_queue.push_front((node_id, 1));
}
let mut count = 0;
while let Some((node_id, depth)) = node_queue.pop_front() {
let stored_node = self.keys.load_node::<BK>(tx, node_id).await?;
if let Node::Internal(_, children) = &stored_node.node {
let depth = depth + 1;
for child_id in children {
node_queue.push_back((*child_id, depth));
}
}
inspect_func(count, depth, node_id, stored_node);
count += 1;
}
Ok(count)
}
}
impl<BK> Node<BK>
where
BK: BKeys,
{
fn debug<F>(&self, to_string: F) -> Result<(), Error>
where
F: Fn(Key) -> Result<String, Error>,
{
match self {
Node::Internal(keys, children) => {
keys.debug(to_string)?;
debug!("Children{:?}", children);
Ok(())
}
Node::Leaf(keys) => keys.debug(to_string),
}
}
}
}

733
lib/src/idx/ft/analyzer/filter.rs Normal file
View file

@ -0,0 +1,733 @@
use crate::idx::ft::analyzer::tokenizer::Tokens;
use crate::sql::filter::Filter as SqlFilter;
use crate::sql::language::Language;
use deunicode::deunicode;
use rust_stemmers::{Algorithm, Stemmer};
pub(super) enum Filter {
Stemmer(Stemmer),
Ascii,
Ngram(u16, u16),
EdgeNgram(u16, u16),
Lowercase,
Uppercase,
}
impl From<SqlFilter> for Filter {
fn from(f: SqlFilter) -> Self {
match f {
SqlFilter::Ascii => Filter::Ascii,
SqlFilter::EdgeNgram(min, max) => Filter::EdgeNgram(min, max),
SqlFilter::Lowercase => Filter::Lowercase,
SqlFilter::Ngram(min, max) => Filter::Ngram(min, max),
SqlFilter::Snowball(l) => {
let a = match l {
Language::Arabic => Stemmer::create(Algorithm::Arabic),
Language::Danish => Stemmer::create(Algorithm::Danish),
Language::Dutch => Stemmer::create(Algorithm::Dutch),
Language::English => Stemmer::create(Algorithm::English),
Language::French => Stemmer::create(Algorithm::French),
Language::German => Stemmer::create(Algorithm::German),
Language::Greek => Stemmer::create(Algorithm::Greek),
Language::Hungarian => Stemmer::create(Algorithm::Hungarian),
Language::Italian => Stemmer::create(Algorithm::Italian),
Language::Norwegian => Stemmer::create(Algorithm::Norwegian),
Language::Portuguese => Stemmer::create(Algorithm::Portuguese),
Language::Romanian => Stemmer::create(Algorithm::Romanian),
Language::Russian => Stemmer::create(Algorithm::Russian),
Language::Spanish => Stemmer::create(Algorithm::Spanish),
Language::Swedish => Stemmer::create(Algorithm::Swedish),
Language::Tamil => Stemmer::create(Algorithm::Tamil),
Language::Turkish => Stemmer::create(Algorithm::Turkish),
};
Filter::Stemmer(a)
}
SqlFilter::Uppercase => Filter::Uppercase,
}
}
}
impl Filter {
pub(super) fn from(f: Option<Vec<SqlFilter>>) -> Option<Vec<Filter>> {
if let Some(f) = f {
let mut r = Vec::with_capacity(f.len());
for f in f {
r.push(f.into());
}
Some(r)
} else {
None
}
}
pub(super) fn apply_filters(mut t: Tokens, f: &Option<Vec<Filter>>) -> Tokens {
if let Some(f) = f {
for f in f {
t = t.filter(f);
}
}
t
}
pub(super) fn apply_filter(&self, c: &str) -> FilterResult {
match self {
Filter::Ascii => Self::deunicode(c),
Filter::EdgeNgram(min, max) => Self::edgengram(c, *min, *max),
Filter::Lowercase => Self::lowercase(c),
Filter::Ngram(min, max) => Self::ngram(c, *min, *max),
Filter::Stemmer(s) => Self::stem(s, c),
Filter::Uppercase => Self::uppercase(c),
}
}
#[inline]
fn check_term(c: &str, s: String) -> FilterResult {
if s.is_empty() {
FilterResult::Ignore
} else if s.eq(c) {
FilterResult::Term(Term::Unchanged)
} else {
FilterResult::Term(Term::NewTerm(s))
}
}
#[inline]
fn lowercase(c: &str) -> FilterResult {
Self::check_term(c, c.to_lowercase())
}
#[inline]
fn uppercase(c: &str) -> FilterResult {
Self::check_term(c, c.to_lowercase())
}
#[inline]
fn deunicode(c: &str) -> FilterResult {
Self::check_term(c, deunicode(c))
}
#[inline]
fn stem(s: &Stemmer, c: &str) -> FilterResult {
Self::check_term(c, s.stem(&c.to_lowercase()).into())
}
#[inline]
fn ngram(c: &str, min: u16, max: u16) -> FilterResult {
let min = min as usize;
let c: Vec<char> = c.chars().collect();
let l = c.len();
if l < min {
return FilterResult::Ignore;
}
let mut ng = vec![];
let r1 = 0..(l - min);
let max = max as usize;
for s in r1 {
let mut e = s + max;
if e > l {
e = l;
}
let r2 = (s + min)..(e + 1);
for p in r2 {
let n = &c[s..p];
if c.eq(n) {
ng.push(Term::Unchanged);
} else {
ng.push(Term::NewTerm(n.into_iter().collect()));
}
}
}
FilterResult::Terms(ng)
}
#[inline]
fn edgengram(c: &str, min: u16, max: u16) -> FilterResult {
let min = min as usize;
let c: Vec<char> = c.chars().collect();
let l = c.len();
if l < min {
return FilterResult::Ignore;
}
let mut max = max as usize;
if max > l {
max = l;
}
let mut ng = vec![];
let r = min..(max + 1);
for p in r {
let n = &c[0..p];
if c.eq(n) {
ng.push(Term::Unchanged);
} else {
ng.push(Term::NewTerm(n.into_iter().collect()));
}
}
FilterResult::Terms(ng)
}
}
pub(super) enum FilterResult {
Ignore,
Term(Term),
Terms(Vec<Term>),
}
pub(super) enum Term {
Unchanged,
NewTerm(String),
}
#[cfg(test)]
mod tests {
use crate::idx::ft::analyzer::tests::test_analyser;
#[test]
fn test_arabic_stemmer() {
let input =
"الكلاب تحب الجري في الحديقة، لكن كلبي الصغير يفضل النوم في سريره بدلاً من الجري";
let output = vec![
"كلاب", "تحب", "الجر", "في", "حديق", "لكن", "كلب", "صغير", "يفضل", "نوم", "في", "سرير",
"بدل", "من", "الجر",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(arabic);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(ar);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(ara);",
input,
&output,
);
}
#[test]
fn test_danish_stemmer() {
let input = "Hunde elsker at løbe i parken, men min lille hund foretrækker at sove i sin kurv frem for at løbe.";
let output = vec![
"hund",
"elsk",
"at",
"løb",
"i",
"park",
",",
"men",
"min",
"lil",
"hund",
"foretræk",
"at",
"sov",
"i",
"sin",
"kurv",
"frem",
"for",
"at",
"løb",
".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(danish);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(dan);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(da);",
input,
&output,
);
}
#[test]
fn test_dutch_stemmer() {
let input = "Honden houden ervan om in het park te rennen, maar mijn kleine hond slaapt liever in zijn mand dan te rennen.";
let output = vec![
"hond", "houd", "ervan", "om", "in", "het", "park", "te", "renn", ",", "mar", "mijn",
"klein", "hond", "slaapt", "liever", "in", "zijn", "mand", "dan", "te", "renn", ".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(dutch);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(nl);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(nld);",
input,
&output,
);
}
#[test]
fn test_english_stemmer() {
let input = "Teachers are often teaching, but my favorite teacher prefers reading in her spare time rather than teaching.";
let output = vec![
"teacher", "are", "often", "teach", ",", "but", "my", "favorit", "teacher", "prefer",
"read", "in", "her", "spare", "time", "rather", "than", "teach", ".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(english);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(eng);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(en);",
input,
&output,
);
}
#[test]
fn test_french_stemmer() {
let input = "Les chiens adorent courir dans le parc, mais mon petit chien aime plutôt se blottir sur le canapé que de courir";
let output = [
"le", "chien", "adorent", "cour", "dan", "le", "parc", ",", "mais", "mon", "pet",
"chien", "aim", "plutôt", "se", "blott", "sur", "le", "canap", "que", "de", "cour",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(french);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(fr);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(fra);",
input,
&output,
);
}
#[test]
fn test_german_stemmer() {
let input = "Hunde lieben es, im Park zu laufen, aber mein kleiner Hund zieht es vor, auf dem Sofa zu schlafen, statt zu laufen.";
let output = [
"hund", "lieb", "es", ",", "im", "park", "zu", "lauf", ",", "aber", "mein", "klein",
"hund", "zieht", "es", "vor", ",", "auf", "dem", "sofa", "zu", "schlaf", ",", "statt",
"zu", "lauf", ".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(german);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(de);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(deu);",
input,
&output,
);
}
#[test]
fn test_greek_stemmer() {
let input = "Τα σκυλιά αγαπούν να τρέχουν στο πάρκο, αλλά ο μικρός μου σκύλος προτιμά να κοιμάται στο κρεβάτι του αντί να τρέχει.";
let output = [
"τα",
"σκυλ",
"αγαπ",
"να",
"τρεχ",
"στ",
"παρκ",
",",
"αλλ",
"ο",
"μικρ",
"μ",
"σκυλ",
"προτιμ",
"να",
"κοιμ",
"στ",
"κρεβατ",
"τ",
"αντ",
"να",
"τρεχ",
".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(greek);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(ell);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(el);",
input,
&output,
);
}
#[test]
fn test_hungarian_stemmer() {
let input = "A kutyák szeretnek futni a parkban, de az én kicsi kutyám inkább alszik a kosarában, mintsem fut.";
let output = [
"a", "kutya", "szeret", "futn", "a", "par", ",", "de", "az", "én", "kics", "kutya",
"inkább", "alsz", "a", "kosar", ",", "mints", "fu", ".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(hungarian);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(hu);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(hun);",
input,
&output,
);
}
#[test]
fn test_italian_stemmer() {
let input = "I cani amano correre nel parco, ma il mio piccolo cane preferisce dormire nel suo cesto piuttosto che correre.";
let output = [
"i", "can", "aman", "corr", "nel", "parc", ",", "ma", "il", "mio", "piccol", "can",
"prefer", "dorm", "nel", "suo", "cest", "piuttost", "che", "corr", ".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(italian);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(it);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(ita);",
input,
&output,
);
}
#[test]
fn test_norwegian_stemmer() {
let input = "Hunder elsker å løpe i parken, men min lille hund foretrekker å sove i sengen sin heller enn å løpe.";
let output = [
"hund",
"elsk",
"å",
"løp",
"i",
"park",
",",
"men",
"min",
"lill",
"hund",
"foretrekk",
"å",
"sov",
"i",
"seng",
"sin",
"hell",
"enn",
"å",
"løp",
".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(norwegian);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(no);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(nor);",
input,
&output,
);
}
#[test]
fn test_portuguese_stemmer() {
let input = "Os cães adoram correr no parque, mas o meu pequeno cão prefere dormir na sua cama em vez de correr.";
let output = [
"os", "cã", "ador", "corr", "no", "parqu", ",", "mas", "o", "meu", "pequen", "cã",
"prefer", "dorm", "na", "sua", "cam", "em", "vez", "de", "corr", ".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(portuguese);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(pt);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(por);",
input,
&output,
);
}
#[test]
fn test_romanian_stemmer() {
let input = "Câinii adoră să alerge în parc, dar cățelul meu preferă să doarmă în coșul lui decât să alerge.";
let output = [
"câin", "ador", "să", "alerg", "în", "parc", ",", "dar", "cățel", "meu", "prefer",
"să", "doarm", "în", "coș", "lui", "decât", "să", "alerg", ".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(romanian);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(ro);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(ron);",
input,
&output,
);
}
#[test]
fn test_russian_stemmer() {
let input = "Собаки любят бегать в парке, но моя маленькая собака предпочитает спать в своей корзине, а не бегать.";
let output = [
"собак",
"люб",
"бега",
"в",
"парк",
",",
"но",
"мо",
"маленьк",
"собак",
"предпочита",
"спат",
"в",
"сво",
"корзин",
",",
"а",
"не",
"бега",
".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(russian);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(ru);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(rus);",
input,
&output,
);
}
#[test]
fn test_spanish_stemmer() {
let input = "Los perros aman correr en el parque, pero mi pequeño perro prefiere dormir en su cama en lugar de correr.";
let output = [
"los", "perr", "aman", "corr", "en", "el", "parqu", ",", "per", "mi", "pequeñ", "perr",
"prefier", "dorm", "en", "su", "cam", "en", "lug", "de", "corr", ".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(spanish);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(es);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(spa);",
input,
&output,
);
}
#[test]
fn test_swedish_stemmer() {
let input = "Hundar älskar att springa i parken, men min lilla hund föredrar att sova i sin säng istället för att springa.";
let output = [
"hund",
"älsk",
"att",
"spring",
"i",
"park",
",",
"men",
"min",
"lill",
"hund",
"föredr",
"att",
"sov",
"i",
"sin",
"säng",
"istället",
"för",
"att",
"spring",
".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(swedish);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(sv);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(swe);",
input,
&output,
);
}
#[test]
fn test_tamil_stemmer() {
let input = "நாய்கள் பூங்காவில் ஓடுவதை விரும்புகின்றன, ஆனால் என் சிறிய நாய் அதன் படுகையில் தூங்குவதை விரும்புகின்றது, ஓட இல்லை.";
let output = [
"ந\u{bbe}ய",
"கள",
"பூங",
"க\u{bbe}வில",
"ஓடுவதை",
"விரும",
"புகி",
"றன",
",",
"ஆன\u{bbe}ல",
"என",
"சிறி",
"ந\u{bbe}ய",
"அதன",
"படுகையில",
"தூங",
"குவதை",
"விரும",
"புகி",
"றது",
",",
"ஓட",
"இல",
"லை",
".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(tamil);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(ta);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(tam);",
input,
&output,
);
}
#[test]
fn test_turkish_stemmer() {
let input = "Köpekler parkta koşmayı sever, ama benim küçük köpeğim koşmaktansa yatağında uyumayı tercih eder.";
let output = [
"köpek", "park", "koşma", "sever", ",", "am", "be", "küçük", "köpek", "koşmak",
"yatak", "uyuma", "tercih", "eder", ".",
];
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(turkish);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(tr);",
input,
&output,
);
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS snowball(tur);",
input,
&output,
);
}
#[test]
fn test_ngram() {
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS lowercase,ngram(2,3);",
"Ālea iacta est",
&vec!["āl", "āle", "le", "lea", "ia", "iac", "ac", "act", "ct", "cta", "es", "est"],
);
}
#[test]
fn test_edgengram() {
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS lowercase,edgengram(2,3);",
"Ālea iacta est",
&vec!["āl", "āle", "ia", "iac", "es", "est"],
);
}
}

127
lib/src/idx/ft/analyzer/mod.rs Normal file
View file

@ -0,0 +1,127 @@
use crate::err::Error;
use crate::idx::ft::analyzer::tokenizer::{Tokenizer, Tokens};
use crate::idx::ft::doclength::DocLength;
use crate::idx::ft::postings::TermFrequency;
use crate::idx::ft::terms::{TermId, Terms};
use crate::kvs::Transaction;
use crate::sql::statements::DefineAnalyzerStatement;
use crate::sql::tokenizer::Tokenizer as SqlTokenizer;
use crate::sql::Array;
use filter::Filter;
use std::collections::hash_map::Entry;
use std::collections::{HashMap, HashSet};
mod filter;
mod tokenizer;
pub(crate) struct Analyzers {}
impl Analyzers {
pub(crate) const LIKE: &'static str = "like";
}
pub(super) struct Analyzer {
t: Option<Vec<SqlTokenizer>>,
f: Option<Vec<Filter>>,
}
impl From<DefineAnalyzerStatement> for Analyzer {
fn from(az: DefineAnalyzerStatement) -> Self {
Self {
t: az.tokenizers,
f: Filter::from(az.filters),
}
}
}
impl Analyzer {
pub(super) async fn extract_terms(
&self,
t: &Terms,
tx: &mut Transaction,
query_string: String,
) -> Result<Vec<Option<TermId>>, Error> {
let tokens = self.analyse(query_string);
// We first collect every unique terms
// as it can contains duplicates
let mut terms = HashSet::new();
for token in tokens.list() {
terms.insert(token);
}
// Now we can extract the term ids
let mut res = Vec::with_capacity(terms.len());
for term in terms {
let term_id = t.get_term_id(tx, tokens.get_token_string(term)).await?;
res.push(term_id);
}
Ok(res)
}
/// This method is used for indexing.
/// It will create new term ids for non already existing terms.
pub(super) async fn extract_terms_with_frequencies(
&self,
t: &mut Terms,
tx: &mut Transaction,
field_content: &Array,
) -> Result<(DocLength, Vec<(TermId, TermFrequency)>), Error> {
let mut doc_length = 0;
// Let's first collect all the inputs, and collect the tokens.
// We need to store them because everything after is zero-copy
let mut inputs = Vec::with_capacity(field_content.0.len());
for v in &field_content.0 {
let input = v.to_owned().convert_to_string()?;
let tks = self.analyse(input);
inputs.push(tks);
}
// We then collect every unique terms and count the frequency
let mut terms = HashMap::new();
for tokens in &inputs {
for token in tokens.list() {
doc_length += 1;
match terms.entry(tokens.get_token_string(&token)) {
Entry::Vacant(e) => {
e.insert(1);
}
Entry::Occupied(mut e) => {
e.insert(*e.get() + 1);
}
}
}
}
// Now we can extract the term ids
let mut res = Vec::with_capacity(terms.len());
for (term, freq) in terms {
res.push((t.resolve_term_id(tx, term).await?, freq));
}
Ok((doc_length, res))
}
fn analyse(&self, input: String) -> Tokens {
if let Some(t) = &self.t {
if !input.is_empty() {
let t = Tokenizer::tokenize(t, input);
return Filter::apply_filters(t, &self.f);
}
}
Tokens::new(input)
}
}
#[cfg(test)]
mod tests {
use super::Analyzer;
use crate::sql::statements::define::analyzer;
pub(super) fn test_analyser(def: &str, input: &str, expected: &[&str]) {
let (_, az) = analyzer(def).unwrap();
let a: Analyzer = az.into();
let tokens = a.analyse(input.to_string());
let mut res = vec![];
for t in tokens.list() {
res.push(tokens.get_token_string(t));
}
assert_eq!(&res, expected);
}
}

292
lib/src/idx/ft/analyzer/tokenizer.rs Normal file
View file

@ -0,0 +1,292 @@
use crate::idx::ft::analyzer::filter::{Filter, FilterResult, Term};
use crate::sql::tokenizer::Tokenizer as SqlTokenizer;
pub(super) struct Tokens {
/// The input string
i: String,
/// The final list of tokens
t: Vec<Token>,
}
impl Tokens {
pub(super) fn new(i: String) -> Self {
Self {
i,
t: Vec::new(),
}
}
pub(super) fn get_token_string<'a>(&'a self, t: &'a Token) -> &str {
t.get_str(&self.i)
}
pub(super) fn filter(self, f: &Filter) -> Tokens {
let mut tks = Vec::new();
let mut res = vec![];
for t in self.t {
if t.is_empty() {
continue;
}
let c = t.get_str(&self.i);
let r = f.apply_filter(c);
res.push((t, r));
}
for (tk, fr) in res {
match fr {
FilterResult::Term(t) => match t {
Term::Unchanged => tks.push(tk),
Term::NewTerm(s) => tks.push(Token::String(s)),
},
FilterResult::Terms(ts) => {
let mut tk = Some(tk);
for t in ts {
match t {
Term::Unchanged => {
if let Some(tk) = tk.take() {
tks.push(tk)
}
}
Term::NewTerm(s) => tks.push(Token::String(s)),
}
}
}
FilterResult::Ignore => {}
};
}
Tokens {
i: self.i,
t: tks,
}
}
pub(super) fn list(&self) -> &Vec<Token> {
&self.t
}
}
#[derive(Clone, Debug, PartialOrd, PartialEq, Eq, Ord, Hash)]
pub(super) enum Token {
Ref(usize, usize),
String(String),
}
impl Token {
fn is_empty(&self) -> bool {
match self {
Token::Ref(start, end) => start == end,
Token::String(s) => s.is_empty(),
}
}
pub(super) fn get_str<'a>(&'a self, i: &'a str) -> &str {
match self {
Token::Ref(s, e) => &i[*s..*e],
Token::String(s) => s,
}
}
}
pub(super) struct Tokenizer {
splitters: Vec<Splitter>,
}
impl Tokenizer {
pub(in crate::idx::ft) fn new(t: &[SqlTokenizer]) -> Self {
Self {
splitters: t.iter().map(|t| t.into()).collect(),
}
}
fn is_valid(c: char) -> bool {
c.is_alphanumeric() || c.is_ascii_punctuation()
}
fn should_split(&mut self, c: char) -> bool {
let mut res = false;
for s in &mut self.splitters {
if s.should_split(c) {
res = true;
}
}
res
}
pub(super) fn tokenize(t: &Vec<SqlTokenizer>, i: String) -> Tokens {
let mut w = Tokenizer::new(t);
let mut last_pos = 0;
let mut current_pos = 0;
let mut t = Vec::new();
for c in i.chars() {
let is_valid = Self::is_valid(c);
let should_split = w.should_split(c);
if should_split || !is_valid {
// The last pos may be more advanced due to the is_valid process
if last_pos < current_pos {
t.push(Token::Ref(last_pos, current_pos));
}
last_pos = current_pos;
// If the character is not valid for indexing (space, control...)
// Then we increase the last position to the next character
if !is_valid {
last_pos += c.len_utf8();
}
}
current_pos += c.len_utf8();
}
if current_pos != last_pos {
t.push(Token::Ref(last_pos, current_pos));
}
Tokens {
i,
t,
}
}
}
struct Splitter {
t: SqlTokenizer,
state: u8,
}
impl From<&SqlTokenizer> for Splitter {
fn from(t: &SqlTokenizer) -> Self {
Self {
t: t.clone(),
state: 0,
}
}
}
impl Splitter {
fn should_split(&mut self, c: char) -> bool {
match &self.t {
SqlTokenizer::Blank => self.blank_state(c),
SqlTokenizer::Camel => self.camel_state(c),
SqlTokenizer::Class => self.class_state(c),
SqlTokenizer::Punct => self.punct_state(c),
}
}
#[inline]
fn state_check(&mut self, s: u8) -> bool {
if s != self.state {
let res = self.state != 0;
self.state = s;
res
} else {
false
}
}
#[inline]
fn blank_state(&mut self, c: char) -> bool {
let s = if c.is_whitespace() {
1
} else {
9
};
self.state_check(s)
}
#[inline]
fn class_state(&mut self, c: char) -> bool {
let s = if c.is_alphabetic() {
1
} else if c.is_numeric() {
2
} else if c.is_whitespace() {
3
} else if c.is_ascii_punctuation() {
4
} else {
9
};
self.state_check(s)
}
#[inline]
fn punct_state(&mut self, c: char) -> bool {
c.is_ascii_punctuation()
}
#[inline]
fn camel_state(&mut self, c: char) -> bool {
let s = if c.is_lowercase() {
1
} else if c.is_uppercase() {
2
} else {
9
};
if s != self.state {
self.state = s;
s == 2
} else {
false
}
}
}
#[cfg(test)]
mod tests {
use crate::idx::ft::analyzer::tests::test_analyser;
#[test]
fn test_tokenize_blank_class() {
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class FILTERS lowercase",
"Abc12345xYZ DL1809 item123456 978-3-16-148410-0 1HGCM82633A123456",
&[
"abc", "12345", "xyz", "dl", "1809", "item", "123456", "978", "-", "3", "-", "16",
"-", "148410", "-", "0", "1", "hgcm", "82633", "a", "123456",
],
);
}
#[test]
fn test_tokenize_source_code() {
test_analyser(
"DEFINE ANALYZER test TOKENIZERS blank,class,camel,punct FILTERS lowercase",
r#"struct MyRectangle {
// specified by corners
top_left: Point,
bottom_right: Point,
}
static LANGUAGE: &str = "Rust";"#,
&[
"struct",
"my",
"rectangle",
"{",
"/",
"/",
"specified",
"by",
"corners",
"top",
"_",
"left",
":",
"point",
",",
"bottom",
"_",
"right",
":",
"point",
",",
"}",
"static",
"language",
":",
"&",
"str",
"=",
"\"",
"rust",
"\"",
";",
],
);
}
}

14
lib/src/idx/ft/docids.rs
View file

@ -21,7 +21,7 @@ impl DocIds {
pub(super) async fn new(
tx: &mut Transaction,
index_key_base: IndexKeyBase,
default_btree_order: usize,
default_btree_order: u32,
) -> Result<Self, Error> {
let keys = DocIdsKeyProvider {
index_key_base: index_key_base.clone(),
@ -59,6 +59,14 @@ impl DocIds {
doc_id
}
pub(super) async fn get_doc_id(
&self,
tx: &mut Transaction,
doc_key: Key,
) -> Result<Option<DocId>, Error> {
self.btree.search::<TrieKeys>(tx, &doc_key).await
}
/// Returns the doc_id for the given doc_key.
/// If the doc_id does not exists, a new one is created, and associated to the given key.
pub(super) async fn resolve_doc_id(
@ -138,7 +146,7 @@ struct State {
impl SerdeState for State {}
impl State {
fn new(default_btree_order: usize) -> Self {
fn new(default_btree_order: u32) -> Self {
Self {
btree: btree::State::new(default_btree_order),
available_ids: None,
@ -189,7 +197,7 @@ mod tests {
use crate::idx::IndexKeyBase;
use crate::kvs::{Datastore, Transaction};
const BTREE_ORDER: usize = 7;
const BTREE_ORDER: u32 = 7;
async fn get_doc_ids(ds: &Datastore) -> (Transaction, DocIds) {
let mut tx = ds.transaction(true, false).await.unwrap();

4
lib/src/idx/ft/doclength.rs
View file

@ -16,7 +16,7 @@ impl DocLengths {
pub(super) async fn new(
tx: &mut Transaction,
index_key_base: IndexKeyBase,
default_btree_order: usize,
default_btree_order: u32,
) -> Result<Self, Error> {
let keys = DocLengthsKeyProvider {
index_key_base,
@ -93,7 +93,7 @@ mod tests {
#[tokio::test]
async fn test_doc_lengths() {
const BTREE_ORDER: usize = 7;
const BTREE_ORDER: u32 = 7;
let ds = Datastore::new("memory").await.unwrap();

561
lib/src/idx/ft/mod.rs
View file

@ -1,34 +1,35 @@
pub(crate) mod analyzer;
pub(crate) mod docids;
mod doclength;
mod postings;
mod scorer;
mod termdocs;
pub(crate) mod terms;
use crate::err::Error;
use crate::error::Db::AnalyzerError;
use crate::idx::ft::docids::{DocId, DocIds};
use crate::idx::ft::doclength::{DocLength, DocLengths};
use crate::idx::ft::postings::{Postings, TermFrequency};
use crate::idx::ft::terms::Terms;
use crate::idx::ft::analyzer::Analyzer;
use crate::idx::ft::docids::DocIds;
use crate::idx::ft::doclength::DocLengths;
use crate::idx::ft::postings::Postings;
use crate::idx::ft::scorer::{BM25Scorer, Score};
use crate::idx::ft::termdocs::TermDocs;
use crate::idx::ft::terms::{TermId, Terms};
use crate::idx::{btree, IndexKeyBase, SerdeState};
use crate::kvs::{Key, Transaction};
use crate::sql::error::IResult;
use nom::bytes::complete::take_while;
use nom::character::complete::multispace0;
use crate::sql::statements::DefineAnalyzerStatement;
use crate::sql::{Array, Object, Thing, Value};
use roaring::treemap::IntoIter;
use roaring::RoaringTreemap;
use serde::{Deserialize, Serialize};
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::ops::BitAnd;
pub(crate) struct FtIndex {
analyzer: Analyzer,
state_key: Key,
index_key_base: IndexKeyBase,
state: State,
bm25: Bm25Params,
btree_default_order: usize,
}
pub(crate) trait HitVisitor {
fn visit(&mut self, tx: &mut Transaction, doc_key: Key, score: Score);
btree_default_order: u32,
}
#[derive(Clone)]
@ -46,13 +47,24 @@ impl Default for Bm25Params {
}
}
pub(super) struct Statistics {
pub(crate) struct Statistics {
doc_ids: btree::Statistics,
terms: btree::Statistics,
doc_lengths: btree::Statistics,
postings: btree::Statistics,
}
impl From<Statistics> for Value {
fn from(stats: Statistics) -> Self {
let mut res = Object::default();
res.insert("doc_ids".to_owned(), Value::from(stats.doc_ids));
res.insert("terms".to_owned(), Value::from(stats.terms));
res.insert("doc_lengths".to_owned(), Value::from(stats.doc_lengths));
res.insert("postings".to_owned(), Value::from(stats.postings));
Value::from(res)
}
}
#[derive(Default, Serialize, Deserialize)]
struct State {
total_docs_lengths: u128,
@ -61,13 +73,12 @@ struct State {
impl SerdeState for State {}
type Score = f32;
impl FtIndex {
pub(crate) async fn new(
tx: &mut Transaction,
az: DefineAnalyzerStatement,
index_key_base: IndexKeyBase,
btree_default_order: usize,
btree_default_order: u32,
) -> Result<Self, Error> {
let state_key: Key = index_key_base.new_bs_key();
let state: State = if let Some(val) = tx.get(state_key.clone()).await? {
@ -81,6 +92,7 @@ impl FtIndex {
index_key_base,
bm25: Bm25Params::default(),
btree_default_order,
analyzer: az.into(),
})
}
@ -92,6 +104,10 @@ impl FtIndex {
Terms::new(tx, self.index_key_base.clone(), self.btree_default_order).await
}
fn term_docs(&self) -> TermDocs {
TermDocs::new(self.index_key_base.clone())
}
async fn doc_lengths(&self, tx: &mut Transaction) -> Result<DocLengths, Error> {
DocLengths::new(tx, self.index_key_base.clone(), self.btree_default_order).await
}
@ -103,11 +119,11 @@ impl FtIndex {
pub(crate) async fn remove_document(
&mut self,
tx: &mut Transaction,
doc_key: Key,
rid: &Thing,
) -> Result<(), Error> {
// Extract and remove the doc_id (if any)
let mut d = self.doc_ids(tx).await?;
if let Some(doc_id) = d.remove_doc(tx, doc_key).await? {
if let Some(doc_id) = d.remove_doc(tx, rid.into()).await? {
self.state.doc_count -= 1;
// Remove the doc length
@ -123,10 +139,12 @@ impl FtIndex {
// Remove the postings
let mut p = self.postings(tx).await?;
let mut t = self.terms(tx).await?;
let td = self.term_docs();
for term_id in term_list {
p.remove_posting(tx, term_id, doc_id).await?;
// if the term is not present in any document in the index, we can remove it
if p.get_doc_count(tx, term_id).await? == 0 {
let doc_count = td.remove_doc(tx, term_id, doc_id).await?;
if doc_count == 0 {
t.remove_term_id(tx, term_id).await?;
}
}
@ -142,18 +160,18 @@ impl FtIndex {
pub(crate) async fn index_document(
&mut self,
tx: &mut Transaction,
doc_key: Key,
field_content: &str,
rid: &Thing,
field_content: &Array,
) -> Result<(), Error> {
// Resolve the doc_id
let mut d = self.doc_ids(tx).await?;
let resolved = d.resolve_doc_id(tx, doc_key).await?;
let resolved = d.resolve_doc_id(tx, rid.into()).await?;
let doc_id = *resolved.doc_id();
// Extract the doc_lengths, terms en frequencies
let mut t = self.terms(tx).await?;
let (doc_length, terms_and_frequencies) =
Self::extract_sorted_terms_with_frequencies(field_content)?;
self.analyzer.extract_terms_with_frequencies(&mut t, tx, field_content).await?;
// Set the doc length
let mut l = self.doc_lengths(tx).await?;
@ -172,15 +190,16 @@ impl FtIndex {
None
};
// Set the terms postings
let terms = t.resolve_term_ids(tx, terms_and_frequencies).await?;
// Set the terms postings and term docs
let term_docs = self.term_docs();
let mut terms_ids = RoaringTreemap::default();
let mut p = self.postings(tx).await?;
for (term_id, term_freq) in terms {
for (term_id, term_freq) in terms_and_frequencies {
p.update_posting(tx, term_id, doc_id, term_freq).await?;
if let Some(old_term_ids) = &mut old_term_ids {
old_term_ids.remove(term_id);
}
term_docs.set_doc(tx, term_id, doc_id).await?;
terms_ids.insert(term_id);
}
@ -188,8 +207,9 @@ impl FtIndex {
if let Some(old_term_ids) = old_term_ids {
for old_term_id in old_term_ids {
p.remove_posting(tx, old_term_id, doc_id).await?;
let doc_count = term_docs.remove_doc(tx, old_term_id, doc_id).await?;
// if the term does not have anymore postings, we can remove the term
if p.get_doc_count(tx, old_term_id).await? == 0 {
if doc_count == 0 {
t.remove_term_id(tx, old_term_id).await?;
}
}
@ -213,77 +233,77 @@ impl FtIndex {
Ok(())
}
// TODO: This is currently a place holder. It has to be replaced by the analyzer/token/filter logic.
fn extract_sorted_terms_with_frequencies(
input: &str,
) -> Result<(DocLength, HashMap<&str, TermFrequency>), Error> {
let mut doc_length = 0;
let mut terms = HashMap::new();
let mut rest = input;
while !rest.is_empty() {
// Extract the next token
match Self::next_token(rest) {
Ok((remaining_input, token)) => {
if !input.is_empty() {
doc_length += 1;
match terms.entry(token) {
Entry::Vacant(e) => {
e.insert(1);
}
Entry::Occupied(mut e) => {
e.insert(*e.get() + 1);
}
}
}
rest = remaining_input;
}
Err(e) => return Err(AnalyzerError(e.to_string())),
}
}
Ok((doc_length, terms))
}
/// Extracting the next token. The string is left trimmed first.
fn next_token(i: &str) -> IResult<&str, &str> {
let (i, _) = multispace0(i)?;
take_while(|c| c != ' ' && c != '\n' && c != '\t')(i)
}
pub(super) async fn search<V>(
pub(super) async fn search(
&self,
tx: &mut Transaction,
term: &str,
visitor: &mut V,
) -> Result<(), Error>
where
V: HitVisitor + Send,
{
let terms = self.terms(tx).await?;
if let Some(term_id) = terms.get_term_id(tx, term).await? {
let postings = self.postings(tx).await?;
let term_doc_count = postings.get_doc_count(tx, term_id).await?;
let doc_lengths = self.doc_lengths(tx).await?;
let doc_ids = self.doc_ids(tx).await?;
if term_doc_count > 0 {
let mut scorer = BM25Scorer::new(
visitor,
query_string: String,
) -> Result<Option<HitsIterator>, Error> {
let t = self.terms(tx).await?;
let td = self.term_docs();
let terms = self.analyzer.extract_terms(&t, tx, query_string).await?;
let mut hits: Option<RoaringTreemap> = None;
let mut terms_docs = Vec::with_capacity(terms.len());
for term_id in terms {
if let Some(term_id) = term_id {
if let Some(term_docs) = td.get_docs(tx, term_id).await? {
if let Some(h) = hits {
hits = Some(h.bitand(&term_docs));
} else {
hits = Some(term_docs.clone());
}
terms_docs.push((term_id, term_docs));
continue;
}
}
return Ok(None);
}
if let Some(hits) = hits {
if !hits.is_empty() {
let postings = self.postings(tx).await?;
let doc_lengths = self.doc_lengths(tx).await?;
// TODO: Scoring should be optional
let scorer = BM25Scorer::new(
doc_lengths,
doc_ids,
self.state.total_docs_lengths,
self.state.doc_count,
term_doc_count as u64,
self.bm25.clone(),
);
let mut it = postings.new_postings_iterator(term_id);
while let Some((doc_id, term_freq)) = it.next(tx).await? {
scorer.visit(tx, doc_id, term_freq).await?;
let doc_ids = self.doc_ids(tx).await?;
return Ok(Some(HitsIterator::new(
doc_ids,
postings,
hits,
terms_docs,
Some(scorer),
)));
}
}
Ok(None)
}
pub(super) async fn match_id_value(
&self,
tx: &mut Transaction,
thg: &Thing,
term: &str,
) -> Result<bool, Error> {
let doc_key: Key = thg.into();
let doc_ids = self.doc_ids(tx).await?;
if let Some(doc_id) = doc_ids.get_doc_id(tx, doc_key).await? {
let terms = self.terms(tx).await?;
if let Some(term_id) = terms.get_term_id(tx, term).await? {
let postings = self.postings(tx).await?;
if let Some(term_freq) = postings.get_term_frequency(tx, term_id, doc_id).await? {
if term_freq > 0 {
return Ok(true);
}
}
}
}
Ok(())
Ok(false)
}
pub(super) async fn statistics(&self, tx: &mut Transaction) -> Result<Statistics, Error> {
pub(crate) async fn statistics(&self, tx: &mut Transaction) -> Result<Statistics, Error> {
// TODO do parallel execution
Ok(Statistics {
doc_ids: self.doc_ids(tx).await?.statistics(tx).await?,
@ -294,102 +314,113 @@ impl FtIndex {
}
}
struct BM25Scorer<'a, V>
where
V: HitVisitor,
{
visitor: &'a mut V,
doc_lengths: DocLengths,
pub(crate) struct HitsIterator {
doc_ids: DocIds,
average_doc_length: f32,
doc_count: f32,
term_doc_count: f32,
bm25: Bm25Params,
postings: Postings,
iter: IntoIter,
terms_docs: Vec<(TermId, RoaringTreemap)>,
scorer: Option<BM25Scorer>,
}
impl<'a, V> BM25Scorer<'a, V>
where
V: HitVisitor,
{
impl HitsIterator {
fn new(
visitor: &'a mut V,
doc_lengths: DocLengths,
doc_ids: DocIds,
total_docs_length: u128,
doc_count: u64,
term_doc_count: u64,
bm25: Bm25Params,
postings: Postings,
hits: RoaringTreemap,
terms_docs: Vec<(TermId, RoaringTreemap)>,
scorer: Option<BM25Scorer>,
) -> Self {
Self {
visitor,
doc_lengths,
doc_ids,
average_doc_length: (total_docs_length as f32) / (doc_count as f32),
doc_count: doc_count as f32,
term_doc_count: term_doc_count as f32,
bm25,
postings,
iter: hits.into_iter(),
terms_docs,
scorer,
}
}
// https://en.wikipedia.org/wiki/Okapi_BM25
// Including the lower-bounding term frequency normalization (2011 CIKM)
fn compute_bm25_score(&self, term_freq: f32, term_doc_count: f32, doc_length: f32) -> f32 {
// (n(qi) + 0.5)
let denominator = term_doc_count + 0.5;
// (N - n(qi) + 0.5)
let numerator = self.doc_count - term_doc_count + 0.5;
let idf = (numerator / denominator).ln();
if idf.is_nan() || idf <= 0.0 {
return 0.0;
}
let tf_prim = 1.0 + term_freq.ln();
// idf * (k1 + 1)
let numerator = idf * (self.bm25.k1 + 1.0) * tf_prim;
// 1 - b + b * (|D| / avgDL)
let denominator = 1.0 - self.bm25.b + self.bm25.b * (doc_length / self.average_doc_length);
// numerator / (k1 * denominator + 1)
numerator / (self.bm25.k1 * denominator + 1.0)
}
async fn visit(
pub(crate) async fn next(
&mut self,
tx: &mut Transaction,
doc_id: DocId,
term_frequency: TermFrequency,
) -> Result<(), Error> {
if let Some(doc_key) = self.doc_ids.get_doc_key(tx, doc_id).await? {
let doc_length = self.doc_lengths.get_doc_length(tx, doc_id).await?.unwrap_or(0);
let bm25_score = self.compute_bm25_score(
term_frequency as f32,
self.term_doc_count,
doc_length as f32,
);
self.visitor.visit(tx, doc_key, bm25_score);
) -> Result<Option<(Thing, Option<Score>)>, Error> {
loop {
if let Some(doc_id) = self.iter.next() {
if let Some(doc_key) = self.doc_ids.get_doc_key(tx, doc_id).await? {
let score = if let Some(scorer) = &self.scorer {
let mut sc = 0.0;
for (term_id, docs) in &self.terms_docs {
if docs.contains(doc_id) {
if let Some(term_freq) =
self.postings.get_term_frequency(tx, *term_id, doc_id).await?
{
sc += scorer.score(tx, doc_id, docs.len(), term_freq).await?;
}
}
}
Some(sc)
} else {
None
};
return Ok(Some((doc_key.into(), score)));
}
} else {
break;
}
}
Ok(())
Ok(None)
}
}
#[cfg(test)]
mod tests {
use crate::idx::ft::{FtIndex, HitVisitor, Score};
use crate::idx::ft::{FtIndex, HitsIterator, Score};
use crate::idx::IndexKeyBase;
use crate::kvs::{Datastore, Key, Transaction};
use crate::kvs::{Datastore, Transaction};
use crate::sql::statements::define::analyzer;
use crate::sql::{Array, Thing};
use std::collections::HashMap;
use test_log::test;
async fn check_hits(
i: Option<HitsIterator>,
tx: &mut Transaction,
e: Vec<(&Thing, Option<Score>)>,
) {
if let Some(mut i) = i {
let mut map = HashMap::new();
while let Some((k, s)) = i.next(tx).await.unwrap() {
map.insert(k, s);
}
assert_eq!(map.len(), e.len());
for (k, p) in e {
assert_eq!(map.get(k), Some(&p));
}
} else {
panic!("hits is none");
}
}
#[test(tokio::test)]
async fn test_ft_index() {
let ds = Datastore::new("memory").await.unwrap();
let (_, az) = analyzer("DEFINE ANALYZER test TOKENIZERS blank;").unwrap();
let default_btree_order = 5;
let doc1: Thing = ("t", "doc1").into();
let doc2: Thing = ("t", "doc2").into();
let doc3: Thing = ("t", "doc3").into();
{
// Add one document
let mut tx = ds.transaction(true, false).await.unwrap();
let mut fti =
FtIndex::new(&mut tx, IndexKeyBase::default(), default_btree_order).await.unwrap();
fti.index_document(&mut tx, "doc1".into(), "hello the world").await.unwrap();
FtIndex::new(&mut tx, az.clone(), IndexKeyBase::default(), default_btree_order)
.await
.unwrap();
fti.index_document(&mut tx, &doc1, &Array::from(vec!["hello the world"]))
.await
.unwrap();
tx.commit().await.unwrap();
}
@ -397,16 +428,20 @@ mod tests {
// Add two documents
let mut tx = ds.transaction(true, false).await.unwrap();
let mut fti =
FtIndex::new(&mut tx, IndexKeyBase::default(), default_btree_order).await.unwrap();
fti.index_document(&mut tx, "doc2".into(), "a yellow hello").await.unwrap();
fti.index_document(&mut tx, "doc3".into(), "foo bar").await.unwrap();
FtIndex::new(&mut tx, az.clone(), IndexKeyBase::default(), default_btree_order)
.await
.unwrap();
fti.index_document(&mut tx, &doc2, &Array::from(vec!["a yellow hello"])).await.unwrap();
fti.index_document(&mut tx, &doc3, &Array::from(vec!["foo bar"])).await.unwrap();
tx.commit().await.unwrap();
}
{
let mut tx = ds.transaction(true, false).await.unwrap();
let fti =
FtIndex::new(&mut tx, IndexKeyBase::default(), default_btree_order).await.unwrap();
FtIndex::new(&mut tx, az.clone(), IndexKeyBase::default(), default_btree_order)
.await
.unwrap();
// Check the statistics
let statistics = fti.statistics(&mut tx).await.unwrap();
@ -416,74 +451,67 @@ mod tests {
assert_eq!(statistics.doc_lengths.keys_count, 3);
// Search & score
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "hello", &mut visitor).await.unwrap();
visitor.check(vec![("doc1".into(), 0.0), ("doc2".into(), 0.0)]);
let i = fti.search(&mut tx, "hello".to_string()).await.unwrap();
check_hits(i, &mut tx, vec![(&doc1, Some(0.0)), (&doc2, Some(0.0))]).await;
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "world", &mut visitor).await.unwrap();
visitor.check(vec![("doc1".into(), 0.4859746)]);
let i = fti.search(&mut tx, "world".to_string()).await.unwrap();
check_hits(i, &mut tx, vec![(&doc1, Some(0.4859746))]).await;
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "yellow", &mut visitor).await.unwrap();
visitor.check(vec![("doc2".into(), 0.4859746)]);
let i = fti.search(&mut tx, "yellow".to_string()).await.unwrap();
check_hits(i, &mut tx, vec![(&doc2, Some(0.4859746))]).await;
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "foo", &mut visitor).await.unwrap();
visitor.check(vec![("doc3".into(), 0.56902087)]);
let i = fti.search(&mut tx, "foo".to_string()).await.unwrap();
check_hits(i, &mut tx, vec![(&doc3, Some(0.56902087))]).await;
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "bar", &mut visitor).await.unwrap();
visitor.check(vec![("doc3".into(), 0.56902087)]);
let i = fti.search(&mut tx, "bar".to_string()).await.unwrap();
check_hits(i, &mut tx, vec![(&doc3, Some(0.56902087))]).await;
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "dummy", &mut visitor).await.unwrap();
visitor.check(Vec::<(Key, f32)>::new());
let i = fti.search(&mut tx, "dummy".to_string()).await.unwrap();
assert!(i.is_none());
}
{
// Reindex one document
let mut tx = ds.transaction(true, false).await.unwrap();
let mut fti =
FtIndex::new(&mut tx, IndexKeyBase::default(), default_btree_order).await.unwrap();
fti.index_document(&mut tx, "doc3".into(), "nobar foo").await.unwrap();
FtIndex::new(&mut tx, az.clone(), IndexKeyBase::default(), default_btree_order)
.await
.unwrap();
fti.index_document(&mut tx, &doc3, &Array::from(vec!["nobar foo"])).await.unwrap();
tx.commit().await.unwrap();
// We can still find 'foo'
let mut tx = ds.transaction(false, false).await.unwrap();
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "foo", &mut visitor).await.unwrap();
visitor.check(vec![("doc3".into(), 0.56902087)]);
let i = fti.search(&mut tx, "foo".to_string()).await.unwrap();
check_hits(i, &mut tx, vec![(&doc3, Some(0.56902087))]).await;
// We can't anymore find 'bar'
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "bar", &mut visitor).await.unwrap();
visitor.check(vec![]);
let i = fti.search(&mut tx, "bar".to_string()).await.unwrap();
assert!(i.is_none());
// We can now find 'nobar'
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "nobar", &mut visitor).await.unwrap();
visitor.check(vec![("doc3".into(), 0.56902087)]);
let i = fti.search(&mut tx, "nobar".to_string()).await.unwrap();
check_hits(i, &mut tx, vec![(&doc3, Some(0.56902087))]).await;
}
{
// Remove documents
let mut tx = ds.transaction(true, false).await.unwrap();
let mut fti =
FtIndex::new(&mut tx, IndexKeyBase::default(), default_btree_order).await.unwrap();
fti.remove_document(&mut tx, "doc1".into()).await.unwrap();
fti.remove_document(&mut tx, "doc2".into()).await.unwrap();
fti.remove_document(&mut tx, "doc3".into()).await.unwrap();
FtIndex::new(&mut tx, az.clone(), IndexKeyBase::default(), default_btree_order)
.await
.unwrap();
fti.remove_document(&mut tx, &doc1).await.unwrap();
fti.remove_document(&mut tx, &doc2).await.unwrap();
fti.remove_document(&mut tx, &doc3).await.unwrap();
tx.commit().await.unwrap();
let mut tx = ds.transaction(false, false).await.unwrap();
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "hello", &mut visitor).await.unwrap();
visitor.check(vec![]);
let i = fti.search(&mut tx, "hello".to_string()).await.unwrap();
assert!(i.is_none());
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "foo", &mut visitor).await.unwrap();
visitor.check(vec![]);
let i = fti.search(&mut tx, "foo".to_string()).await.unwrap();
assert!(i.is_none());
}
}
@ -495,37 +523,57 @@ mod tests {
// Therefore it makes sense to do multiple runs.
for _ in 0..10 {
let ds = Datastore::new("memory").await.unwrap();
let (_, az) = analyzer("DEFINE ANALYZER test TOKENIZERS blank;").unwrap();
let doc1: Thing = ("t", "doc1").into();
let doc2: Thing = ("t", "doc2").into();
let doc3: Thing = ("t", "doc3").into();
let doc4: Thing = ("t", "doc4").into();
let default_btree_order = 5;
{
let mut tx = ds.transaction(true, false).await.unwrap();
let mut fti = FtIndex::new(&mut tx, IndexKeyBase::default(), default_btree_order)
.await
.unwrap();
let mut fti =
FtIndex::new(&mut tx, az.clone(), IndexKeyBase::default(), default_btree_order)
.await
.unwrap();
fti.index_document(
&mut tx,
"doc1".into(),
"the quick brown fox jumped over the lazy dog",
&doc1,
&Array::from(vec!["the quick brown fox jumped over the lazy dog"]),
)
.await
.unwrap();
fti.index_document(
&mut tx,
&doc2,
&Array::from(vec!["the fast fox jumped over the lazy dog"]),
)
.await
.unwrap();
fti.index_document(
&mut tx,
&doc3,
&Array::from(vec!["the dog sat there and did nothing"]),
)
.await
.unwrap();
fti.index_document(
&mut tx,
&doc4,
&Array::from(vec!["the other animals sat there watching"]),
)
.await
.unwrap();
fti.index_document(&mut tx, "doc2".into(), "the fast fox jumped over the lazy dog")
.await
.unwrap();
fti.index_document(&mut tx, "doc3".into(), "the dog sat there and did nothing")
.await
.unwrap();
fti.index_document(&mut tx, "doc4".into(), "the other animals sat there watching")
.await
.unwrap();
tx.commit().await.unwrap();
}
{
let mut tx = ds.transaction(true, false).await.unwrap();
let fti = FtIndex::new(&mut tx, IndexKeyBase::default(), default_btree_order)
.await
.unwrap();
let fti =
FtIndex::new(&mut tx, az.clone(), IndexKeyBase::default(), default_btree_order)
.await
.unwrap();
let statistics = fti.statistics(&mut tx).await.unwrap();
assert_eq!(statistics.terms.keys_count, 17);
@ -533,70 +581,47 @@ mod tests {
assert_eq!(statistics.doc_ids.keys_count, 4);
assert_eq!(statistics.doc_lengths.keys_count, 4);
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "the", &mut visitor).await.unwrap();
visitor.check(vec![
("doc1".into(), 0.0),
("doc2".into(), 0.0),
("doc3".into(), 0.0),
("doc4".into(), 0.0),
]);
let i = fti.search(&mut tx, "the".to_string()).await.unwrap();
check_hits(
i,
&mut tx,
vec![
(&doc1, Some(0.0)),
(&doc2, Some(0.0)),
(&doc3, Some(0.0)),
(&doc4, Some(0.0)),
],
)
.await;
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "dog", &mut visitor).await.unwrap();
visitor.check(vec![
("doc1".into(), 0.0),
("doc2".into(), 0.0),
("doc3".into(), 0.0),
]);
let i = fti.search(&mut tx, "dog".to_string()).await.unwrap();
check_hits(
i,
&mut tx,
vec![(&doc1, Some(0.0)), (&doc2, Some(0.0)), (&doc3, Some(0.0))],
)
.await;
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "fox", &mut visitor).await.unwrap();
visitor.check(vec![("doc1".into(), 0.0), ("doc2".into(), 0.0)]);
let i = fti.search(&mut tx, "fox".to_string()).await.unwrap();
check_hits(i, &mut tx, vec![(&doc1, Some(0.0)), (&doc2, Some(0.0))]).await;
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "over", &mut visitor).await.unwrap();
visitor.check(vec![("doc1".into(), 0.0), ("doc2".into(), 0.0)]);
let i = fti.search(&mut tx, "over".to_string()).await.unwrap();
check_hits(i, &mut tx, vec![(&doc1, Some(0.0)), (&doc2, Some(0.0))]).await;
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "lazy", &mut visitor).await.unwrap();
visitor.check(vec![("doc1".into(), 0.0), ("doc2".into(), 0.0)]);
let i = fti.search(&mut tx, "lazy".to_string()).await.unwrap();
check_hits(i, &mut tx, vec![(&doc1, Some(0.0)), (&doc2, Some(0.0))]).await;
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "jumped", &mut visitor).await.unwrap();
visitor.check(vec![("doc1".into(), 0.0), ("doc2".into(), 0.0)]);
let i = fti.search(&mut tx, "jumped".to_string()).await.unwrap();
check_hits(i, &mut tx, vec![(&doc1, Some(0.0)), (&doc2, Some(0.0))]).await;
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "nothing", &mut visitor).await.unwrap();
visitor.check(vec![("doc3".into(), 0.87105393)]);
let i = fti.search(&mut tx, "nothing".to_string()).await.unwrap();
check_hits(i, &mut tx, vec![(&doc3, Some(0.87105393))]).await;
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "animals", &mut visitor).await.unwrap();
visitor.check(vec![("doc4".into(), 0.92279965)]);
let i = fti.search(&mut tx, "animals".to_string()).await.unwrap();
check_hits(i, &mut tx, vec![(&doc4, Some(0.92279965))]).await;
let mut visitor = HashHitVisitor::default();
fti.search(&mut tx, "dummy", &mut visitor).await.unwrap();
visitor.check(Vec::<(Key, f32)>::new());
}
}
}
#[derive(Default)]
pub(super) struct HashHitVisitor {
map: HashMap<Key, Score>,
}
impl HitVisitor for HashHitVisitor {
fn visit(&mut self, _tx: &mut Transaction, doc_key: Key, score: Score) {
self.map.insert(doc_key, score);
}
}
impl HashHitVisitor {
pub(super) fn check(&self, res: Vec<(Key, Score)>) {
assert_eq!(res.len(), self.map.len(), "{:?}", self.map);
for (k, p) in res {
assert_eq!(self.map.get(&k), Some(&p));
let i = fti.search(&mut tx, "dummy".to_string()).await.unwrap();
assert!(i.is_none());
}
}
}

84
lib/src/idx/ft/postings.rs
View file

@ -1,10 +1,9 @@
use crate::err::Error;
use crate::idx::bkeys::TrieKeys;
use crate::idx::btree::{BTree, BTreeIterator, KeyProvider, NodeId, Payload, Statistics};
use crate::idx::btree::{BTree, KeyProvider, NodeId, Statistics};
use crate::idx::ft::docids::DocId;
use crate::idx::ft::terms::TermId;
use crate::idx::{btree, IndexKeyBase, SerdeState};
use crate::key::bf::Bf;
use crate::kvs::{Key, Transaction};
pub(super) type TermFrequency = u64;
@ -19,7 +18,7 @@ impl Postings {
pub(super) async fn new(
tx: &mut Transaction,
index_key_base: IndexKeyBase,
default_btree_order: usize,
default_btree_order: u32,
) -> Result<Self, Error> {
let keys = PostingsKeyProvider {
index_key_base: index_key_base.clone(),
@ -48,6 +47,16 @@ impl Postings {
self.btree.insert::<TrieKeys>(tx, key, term_freq).await
}
pub(super) async fn get_term_frequency(
&self,
tx: &mut Transaction,
term_id: TermId,
doc_id: DocId,
) -> Result<Option<TermFrequency>, Error> {
let key = self.index_key_base.new_bf_key(term_id, doc_id);
self.btree.search::<TrieKeys>(tx, &key).await
}
pub(super) async fn remove_posting(
&mut self,
tx: &mut Transaction,
@ -58,25 +67,6 @@ impl Postings {
self.btree.delete::<TrieKeys>(tx, key).await
}
pub(super) fn new_postings_iterator(&self, term_id: TermId) -> PostingsIterator {
let prefix_key = self.index_key_base.new_bf_prefix_key(term_id);
let i = self.btree.search_by_prefix(prefix_key);
PostingsIterator::new(i)
}
pub(super) async fn get_doc_count(
&self,
tx: &mut Transaction,
term_id: TermId,
) -> Result<usize, Error> {
let mut count = 0;
let mut it = self.new_postings_iterator(term_id);
while let Some((_, _)) = it.next(tx).await? {
count += 1;
}
Ok(count)
}
pub(super) async fn statistics(&self, tx: &mut Transaction) -> Result<Statistics, Error> {
self.btree.statistics::<TrieKeys>(tx).await
}
@ -103,56 +93,16 @@ impl KeyProvider for PostingsKeyProvider {
}
}
pub(super) struct PostingsIterator {
btree_iterator: BTreeIterator<PostingsKeyProvider>,
}
impl PostingsIterator {
fn new(btree_iterator: BTreeIterator<PostingsKeyProvider>) -> Self {
Self {
btree_iterator,
}
}
pub(super) async fn next(
&mut self,
tx: &mut Transaction,
) -> Result<Option<(DocId, Payload)>, Error> {
Ok(self.btree_iterator.next::<TrieKeys>(tx).await?.map(|(k, p)| {
let posting_key: Bf = (&k).into();
(posting_key.doc_id, p)
}))
}
}
#[cfg(test)]
mod tests {
use crate::idx::btree::Payload;
use crate::idx::ft::docids::DocId;
use crate::idx::ft::postings::{Postings, PostingsIterator};
use crate::idx::ft::postings::Postings;
use crate::idx::IndexKeyBase;
use crate::kvs::{Datastore, Transaction};
use std::collections::HashMap;
use crate::kvs::Datastore;
use test_log::test;
async fn check_postings(
mut i: PostingsIterator,
tx: &mut Transaction,
e: Vec<(DocId, Payload)>,
) {
let mut map = HashMap::new();
while let Some((d, p)) = i.next(tx).await.unwrap() {
map.insert(d, p);
}
assert_eq!(map.len(), e.len());
for (k, p) in e {
assert_eq!(map.get(&k), Some(&p));
}
}
#[test(tokio::test)]
async fn test_postings() {
const DEFAULT_BTREE_ORDER: usize = 5;
const DEFAULT_BTREE_ORDER: u32 = 5;
let ds = Datastore::new("memory").await.unwrap();
let mut tx = ds.transaction(true, false).await.unwrap();
@ -174,8 +124,8 @@ mod tests {
Postings::new(&mut tx, IndexKeyBase::default(), DEFAULT_BTREE_ORDER).await.unwrap();
assert_eq!(p.statistics(&mut tx).await.unwrap().keys_count, 2);
let i = p.new_postings_iterator(1);
check_postings(i, &mut tx, vec![(2, 3), (4, 5)]).await;
assert_eq!(p.get_term_frequency(&mut tx, 1, 2).await.unwrap(), Some(3));
assert_eq!(p.get_term_frequency(&mut tx, 1, 4).await.unwrap(), Some(5));
// Check removal of doc 2
assert_eq!(p.remove_posting(&mut tx, 1, 2).await.unwrap(), Some(3));

62
lib/src/idx/ft/scorer.rs Normal file
View file

@ -0,0 +1,62 @@
use crate::err::Error;
use crate::idx::ft::docids::DocId;
use crate::idx::ft::doclength::{DocLength, DocLengths};
use crate::idx::ft::postings::TermFrequency;
use crate::idx::ft::Bm25Params;
use crate::kvs::Transaction;
pub(super) type Score = f32;
pub(super) struct BM25Scorer {
doc_lengths: DocLengths,
average_doc_length: f32,
doc_count: f32,
bm25: Bm25Params,
}
impl BM25Scorer {
pub(super) fn new(
doc_lengths: DocLengths,
total_docs_length: u128,
doc_count: u64,
bm25: Bm25Params,
) -> Self {
Self {
doc_lengths,
average_doc_length: (total_docs_length as f32) / (doc_count as f32),
doc_count: doc_count as f32,
bm25,
}
}
pub(super) async fn score(
&self,
tx: &mut Transaction,
doc_id: DocId,
term_doc_count: DocLength,
term_frequency: TermFrequency,
) -> Result<Score, Error> {
let doc_length = self.doc_lengths.get_doc_length(tx, doc_id).await?.unwrap_or(0);
Ok(self.compute_bm25_score(term_frequency as f32, term_doc_count as f32, doc_length as f32))
}
// https://en.wikipedia.org/wiki/Okapi_BM25
// Including the lower-bounding term frequency normalization (2011 CIKM)
fn compute_bm25_score(&self, term_freq: f32, term_doc_count: f32, doc_length: f32) -> f32 {
// (n(qi) + 0.5)
let denominator = term_doc_count + 0.5;
// (N - n(qi) + 0.5)
let numerator = self.doc_count - term_doc_count + 0.5;
let idf = (numerator / denominator).ln();
if idf.is_nan() || idf <= 0.0 {
return 0.0;
}
let tf_prim = 1.0 + term_freq.ln();
// idf * (k1 + 1)
let numerator = idf * (self.bm25.k1 + 1.0) * tf_prim;
// 1 - b + b * (|D| / avgDL)
let denominator = 1.0 - self.bm25.b + self.bm25.b * (doc_length / self.average_doc_length);
// numerator / (k1 * denominator + 1)
numerator / (self.bm25.k1 * denominator + 1.0)
}
}

69
lib/src/idx/ft/termdocs.rs Normal file
View file

@ -0,0 +1,69 @@
use crate::err::Error;
use crate::idx::ft::docids::DocId;
use crate::idx::ft::doclength::DocLength;
use crate::idx::ft::terms::TermId;
use crate::idx::{IndexKeyBase, SerdeState};
use crate::kvs::Transaction;
use roaring::RoaringTreemap;
pub(super) struct TermDocs {
index_key_base: IndexKeyBase,
}
impl TermDocs {
pub(super) fn new(index_key_base: IndexKeyBase) -> Self {
Self {
index_key_base,
}
}
pub(super) async fn set_doc(
&self,
tx: &mut Transaction,
term_id: TermId,
doc_id: DocId,
) -> Result<(), Error> {
let mut docs = self.get_docs(tx, term_id).await?.unwrap_or_else(RoaringTreemap::new);
if docs.insert(doc_id) {
let key = self.index_key_base.new_bc_key(term_id);
tx.set(key, docs.try_to_val()?).await?;
}
Ok(())
}
pub(super) async fn get_docs(
&self,
tx: &mut Transaction,
term_id: TermId,
) -> Result<Option<RoaringTreemap>, Error> {
let key = self.index_key_base.new_bc_key(term_id);
if let Some(val) = tx.get(key).await? {
let docs = RoaringTreemap::try_from_val(val)?;
Ok(Some(docs))
} else {
Ok(None)
}
}
pub(super) async fn remove_doc(
&self,
tx: &mut Transaction,
term_id: TermId,
doc_id: DocId,
) -> Result<DocLength, Error> {
if let Some(mut docs) = self.get_docs(tx, term_id).await? {
if docs.contains(doc_id) {
docs.remove(doc_id);
let key = self.index_key_base.new_bc_key(term_id);
if docs.is_empty() {
tx.del(key).await?;
} else {
tx.set(key, docs.try_to_val()?).await?;
}
}
Ok(docs.len())
} else {
Ok(0)
}
}
}

77
lib/src/idx/ft/terms.rs
View file

@ -1,12 +1,10 @@
use crate::err::Error;
use crate::idx::bkeys::FstKeys;
use crate::idx::btree::{BTree, KeyProvider, NodeId, Statistics};
use crate::idx::ft::postings::TermFrequency;
use crate::idx::{btree, IndexKeyBase, SerdeState};
use crate::kvs::{Key, Transaction};
use roaring::RoaringTreemap;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
pub(crate) type TermId = u64;
@ -23,7 +21,7 @@ impl Terms {
pub(super) async fn new(
tx: &mut Transaction,
index_key_base: IndexKeyBase,
default_btree_order: usize,
default_btree_order: u32,
) -> Result<Self, Error> {
let keys = TermsKeyProvider {
index_key_base: index_key_base.clone(),
@ -61,19 +59,11 @@ impl Terms {
term_id
}
pub(super) async fn resolve_term_ids(
pub(super) async fn resolve_term_id(
&mut self,
tx: &mut Transaction,
terms_frequencies: HashMap<&str, TermFrequency>,
) -> Result<HashMap<TermId, TermFrequency>, Error> {
let mut res = HashMap::with_capacity(terms_frequencies.len());
for (term, freq) in terms_frequencies {
res.insert(self.resolve_term_id(tx, term).await?, freq);
}
Ok(res)
}
async fn resolve_term_id(&mut self, tx: &mut Transaction, term: &str) -> Result<TermId, Error> {
term: &str,
) -> Result<TermId, Error> {
let term_key = term.into();
if let Some(term_id) = self.btree.search::<FstKeys>(tx, &term_key).await? {
Ok(term_id)
@ -144,7 +134,7 @@ struct State {
impl SerdeState for State {}
impl State {
fn new(default_btree_order: usize) -> Self {
fn new(default_btree_order: u32) -> Self {
Self {
btree: btree::State::new(default_btree_order),
available_ids: None,
@ -174,7 +164,7 @@ mod tests {
use crate::idx::IndexKeyBase;
use crate::kvs::Datastore;
use rand::{thread_rng, Rng};
use std::collections::{HashMap, HashSet};
use std::collections::HashSet;
fn random_term(key_length: usize) -> String {
thread_rng()
@ -194,7 +184,7 @@ mod tests {
#[tokio::test]
async fn test_resolve_terms() {
const BTREE_ORDER: usize = 7;
const BTREE_ORDER: u32 = 7;
let idx = IndexKeyBase::default();
@ -208,50 +198,45 @@ mod tests {
// Resolve a first term
let mut tx = ds.transaction(true, false).await.unwrap();
let mut t = Terms::new(&mut tx, idx.clone(), BTREE_ORDER).await.unwrap();
let res = t.resolve_term_ids(&mut tx, HashMap::from([("C", 103)])).await.unwrap();
assert_eq!(t.resolve_term_id(&mut tx, "C").await.unwrap(), 0);
assert_eq!(t.statistics(&mut tx).await.unwrap().keys_count, 1);
t.finish(&mut tx).await.unwrap();
tx.commit().await.unwrap();
assert_eq!(res, HashMap::from([(0, 103)]));
// Resolve a second term
let mut tx = ds.transaction(true, false).await.unwrap();
let mut t = Terms::new(&mut tx, idx.clone(), BTREE_ORDER).await.unwrap();
let res = t.resolve_term_ids(&mut tx, HashMap::from([("D", 104)])).await.unwrap();
assert_eq!(t.resolve_term_id(&mut tx, "D").await.unwrap(), 1);
assert_eq!(t.statistics(&mut tx).await.unwrap().keys_count, 2);
t.finish(&mut tx).await.unwrap();
tx.commit().await.unwrap();
assert_eq!(res, HashMap::from([(1, 104)]));
// Resolve two existing terms with new frequencies
let mut tx = ds.transaction(true, false).await.unwrap();
let mut t = Terms::new(&mut tx, idx.clone(), BTREE_ORDER).await.unwrap();
let res =
t.resolve_term_ids(&mut tx, HashMap::from([("C", 113), ("D", 114)])).await.unwrap();
assert_eq!(t.resolve_term_id(&mut tx, "C").await.unwrap(), 0);
assert_eq!(t.resolve_term_id(&mut tx, "D").await.unwrap(), 1);
assert_eq!(t.statistics(&mut tx).await.unwrap().keys_count, 2);
t.finish(&mut tx).await.unwrap();
tx.commit().await.unwrap();
assert_eq!(res, HashMap::from([(0, 113), (1, 114)]));
// Resolve one existing terms and two new terms
let mut tx = ds.transaction(true, false).await.unwrap();
let mut t = Terms::new(&mut tx, idx.clone(), BTREE_ORDER).await.unwrap();
let res = t
.resolve_term_ids(&mut tx, HashMap::from([("A", 101), ("C", 123), ("E", 105)]))
.await
.unwrap();
assert_eq!(t.resolve_term_id(&mut tx, "A").await.unwrap(), 2);
assert_eq!(t.resolve_term_id(&mut tx, "C").await.unwrap(), 0);
assert_eq!(t.resolve_term_id(&mut tx, "E").await.unwrap(), 3);
assert_eq!(t.statistics(&mut tx).await.unwrap().keys_count, 4);
t.finish(&mut tx).await.unwrap();
tx.commit().await.unwrap();
assert!(
res.eq(&HashMap::from([(3, 101), (0, 123), (2, 105)]))
|| res.eq(&HashMap::from([(2, 101), (0, 123), (3, 105)]))
);
}
#[tokio::test]
async fn test_deletion() {
const BTREE_ORDER: usize = 7;
const BTREE_ORDER: u32 = 7;
let idx = IndexKeyBase::default();
@ -264,9 +249,9 @@ mod tests {
assert!(t.remove_term_id(&mut tx, 0).await.is_ok());
// Create few terms
t.resolve_term_ids(&mut tx, HashMap::from([("A", 101), ("C", 123), ("E", 105)]))
.await
.unwrap();
t.resolve_term_id(&mut tx, "A").await.unwrap();
t.resolve_term_id(&mut tx, "C").await.unwrap();
t.resolve_term_id(&mut tx, "E").await.unwrap();
for term in ["A", "C", "E"] {
let term_id = t.get_term_id(&mut tx, term).await.unwrap();
@ -279,12 +264,8 @@ mod tests {
}
// Check id recycling
let res =
t.resolve_term_ids(&mut tx, HashMap::from([("B", 102), ("D", 104)])).await.unwrap();
assert!(
res.eq(&HashMap::from([(0, 102), (1, 104)]))
|| res.eq(&HashMap::from([(0, 104), (1, 102)]))
);
assert_eq!(t.resolve_term_id(&mut tx, "B").await.unwrap(), 0);
assert_eq!(t.resolve_term_id(&mut tx, "D").await.unwrap(), 1);
t.finish(&mut tx).await.unwrap();
tx.commit().await.unwrap();
@ -307,9 +288,9 @@ mod tests {
let mut tx = ds.transaction(true, false).await.unwrap();
let mut t = Terms::new(&mut tx, IndexKeyBase::default(), 100).await.unwrap();
let terms_string = random_term_freq_vec(50);
let terms_str: HashMap<&str, TermFrequency> =
terms_string.iter().map(|(t, f)| (t.as_str(), *f)).collect();
t.resolve_term_ids(&mut tx, terms_str).await.unwrap();
for (term, _) in terms_string {
t.resolve_term_id(&mut tx, &term).await.unwrap();
}
t.finish(&mut tx).await.unwrap();
tx.commit().await.unwrap();
}
@ -323,9 +304,9 @@ mod tests {
let mut t = Terms::new(&mut tx, IndexKeyBase::default(), 100).await.unwrap();
for _ in 0..10 {
let terms_string = random_term_freq_vec(50);
let terms_str: HashMap<&str, TermFrequency> =
terms_string.iter().map(|(t, f)| (t.as_str(), *f)).collect();
t.resolve_term_ids(&mut tx, terms_str).await.unwrap();
for (term, _) in terms_string {
t.resolve_term_id(&mut tx, &term).await.unwrap();
}
}
t.finish(&mut tx).await.unwrap();
tx.commit().await.unwrap();

130
lib/src/idx/mod.rs
View file

@ -1,14 +1,16 @@
mod bkeys;
pub(crate) mod btree;
pub(crate) mod ft;
pub(crate) mod planner;
use crate::dbs::Options;
use crate::err::Error;
use crate::idx::btree::NodeId;
use crate::idx::ft::docids::DocId;
use crate::idx::ft::terms::TermId;
use crate::key::bc::Bc;
use crate::key::bd::Bd;
use crate::key::bf::{Bf, BfPrefix};
use crate::key::bf::Bf;
use crate::key::bi::Bi;
use crate::key::bk::Bk;
use crate::key::bl::Bl;
@ -20,10 +22,16 @@ use crate::kvs::{Key, Val};
use crate::sql::statements::DefineIndexStatement;
use roaring::RoaringTreemap;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use serde::Serialize;
use std::sync::Arc;
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Default)]
pub(crate) struct IndexKeyBase {
inner: Arc<Inner>,
}
#[derive(Debug, Default)]
struct Inner {
ns: String,
db: String,
tb: String,
@ -33,73 +41,123 @@ pub(crate) struct IndexKeyBase {
impl IndexKeyBase {
pub(crate) fn new(opt: &Options, ix: &DefineIndexStatement) -> Self {
Self {
ns: opt.ns().to_string(),
db: opt.db().to_string(),
tb: ix.what.to_string(),
ix: ix.name.to_string(),
inner: Arc::new(Inner {
ns: opt.ns().to_string(),
db: opt.db().to_string(),
tb: ix.what.to_string(),
ix: ix.name.to_string(),
}),
}
}
fn new_bc_key(&self, term_id: TermId) -> Key {
Bc::new(
self.inner.ns.as_str(),
self.inner.db.as_str(),
self.inner.tb.as_str(),
self.inner.ix.as_str(),
term_id,
)
.into()
}
fn new_bd_key(&self, node_id: Option<NodeId>) -> Key {
Bd::new(self.ns.as_str(), self.db.as_str(), self.tb.as_str(), self.ix.as_str(), node_id)
.into()
Bd::new(
self.inner.ns.as_str(),
self.inner.db.as_str(),
self.inner.tb.as_str(),
self.inner.ix.as_str(),
node_id,
)
.into()
}
fn new_bi_key(&self, doc_id: DocId) -> Key {
Bi::new(self.ns.as_str(), self.db.as_str(), self.tb.as_str(), self.ix.as_str(), doc_id)
.into()
Bi::new(
self.inner.ns.as_str(),
self.inner.db.as_str(),
self.inner.tb.as_str(),
self.inner.ix.as_str(),
doc_id,
)
.into()
}
fn new_bk_key(&self, doc_id: DocId) -> Key {
Bk::new(self.ns.as_str(), self.db.as_str(), self.tb.as_str(), self.ix.as_str(), doc_id)
.into()
Bk::new(
self.inner.ns.as_str(),
self.inner.db.as_str(),
self.inner.tb.as_str(),
self.inner.ix.as_str(),
doc_id,
)
.into()
}
fn new_bl_key(&self, node_id: Option<NodeId>) -> Key {
Bl::new(self.ns.as_str(), self.db.as_str(), self.tb.as_str(), self.ix.as_str(), node_id)
.into()
Bl::new(
self.inner.ns.as_str(),
self.inner.db.as_str(),
self.inner.tb.as_str(),
self.inner.ix.as_str(),
node_id,
)
.into()
}
fn new_bp_key(&self, node_id: Option<NodeId>) -> Key {
Bp::new(self.ns.as_str(), self.db.as_str(), self.tb.as_str(), self.ix.as_str(), node_id)
.into()
Bp::new(
self.inner.ns.as_str(),
self.inner.db.as_str(),
self.inner.tb.as_str(),
self.inner.ix.as_str(),
node_id,
)
.into()
}
fn new_bf_key(&self, term_id: TermId, doc_id: DocId) -> Key {
Bf::new(
self.ns.as_str(),
self.db.as_str(),
self.tb.as_str(),
self.ix.as_str(),
self.inner.ns.as_str(),
self.inner.db.as_str(),
self.inner.tb.as_str(),
self.inner.ix.as_str(),
term_id,
doc_id,
)
.into()
}
fn new_bf_prefix_key(&self, term_id: TermId) -> Key {
BfPrefix::new(
self.ns.as_str(),
self.db.as_str(),
self.tb.as_str(),
self.ix.as_str(),
term_id,
fn new_bt_key(&self, node_id: Option<NodeId>) -> Key {
Bt::new(
self.inner.ns.as_str(),
self.inner.db.as_str(),
self.inner.tb.as_str(),
self.inner.ix.as_str(),
node_id,
)
.into()
}
fn new_bt_key(&self, node_id: Option<NodeId>) -> Key {
Bt::new(self.ns.as_str(), self.db.as_str(), self.tb.as_str(), self.ix.as_str(), node_id)
.into()
}
fn new_bs_key(&self) -> Key {
Bs::new(self.ns.as_str(), self.db.as_str(), self.tb.as_str(), self.ix.as_str()).into()
Bs::new(
self.inner.ns.as_str(),
self.inner.db.as_str(),
self.inner.tb.as_str(),
self.inner.ix.as_str(),
)
.into()
}
fn new_bu_key(&self, term_id: TermId) -> Key {
Bu::new(self.ns.as_str(), self.db.as_str(), self.tb.as_str(), self.ix.as_str(), term_id)
.into()
Bu::new(
self.inner.ns.as_str(),
self.inner.db.as_str(),
self.inner.tb.as_str(),
self.inner.ix.as_str(),
term_id,
)
.into()
}
}

97
lib/src/idx/planner/executor.rs Normal file
View file

@ -0,0 +1,97 @@
use crate::dbs::{Options, Transaction};
use crate::err::Error;
use crate::idx::ft::FtIndex;
use crate::idx::planner::tree::IndexMap;
use crate::idx::IndexKeyBase;
use crate::sql::index::Index;
use crate::sql::{Expression, Table, Thing, Value};
use std::collections::HashMap;
use std::sync::Arc;
#[derive(Clone)]
pub(crate) struct QueryExecutor {
inner: Arc<Inner>,
}
struct Inner {
table: String,
index_map: IndexMap,
pre_match: Option<Expression>,
ft_map: HashMap<String, FtIndex>,
}
impl QueryExecutor {
pub(super) async fn new(
opt: &Options,
txn: &Transaction,
table: &Table,
index_map: IndexMap,
pre_match: Option<Expression>,
) -> Result<Self, Error> {
let mut run = txn.lock().await;
let mut ft_map = HashMap::new();
for ios in index_map.values() {
for io in ios {
if let Index::Search {
az,
order,
..
} = &io.ix.index
{
if !ft_map.contains_key(&io.ix.name.0) {
let ikb = IndexKeyBase::new(opt, &io.ix);
let az = run.get_az(opt.ns(), opt.db(), az.as_str()).await?;
let ft = FtIndex::new(&mut run, az, ikb, *order).await?;
ft_map.insert(io.ix.name.0.clone(), ft);
}
}
}
}
Ok(Self {
inner: Arc::new(Inner {
table: table.0.clone(),
index_map,
pre_match,
ft_map,
}),
})
}
pub(crate) async fn matches(
&self,
txn: &Transaction,
thg: &Thing,
exp: &Expression,
) -> Result<Value, Error> {
// If we find the expression in `pre_match`,
// it means that we are using an Iterator::Index
// and we are iterating over document that already matches the expression.
if let Some(pre_match) = &self.inner.pre_match {
if pre_match.eq(exp) {
return Ok(Value::Bool(true));
}
}
// Otherwise, we look for the first possible index options, and evaluate the expression
// Does the record id match this executor's table?
if thg.tb.eq(&self.inner.table) {
if let Some(ios) = self.inner.index_map.get(exp) {
for io in ios {
if let Some(fti) = self.inner.ft_map.get(&io.ix.name.0) {
let mut run = txn.lock().await;
// TODO The query string could be extracted when IndexOptions are created
let query_string = io.v.clone().convert_to_string()?;
return Ok(Value::Bool(
fti.match_id_value(&mut run, thg, &query_string).await?,
));
}
}
}
}
// If no previous case were successful, we end up with a user error
Err(Error::NoIndexFoundForMatch {
value: exp.to_string(),
})
}
}

106
lib/src/idx/planner/mod.rs Normal file
View file

@ -0,0 +1,106 @@
pub(crate) mod executor;
pub(crate) mod plan;
mod tree;
use crate::ctx::Context;
use crate::dbs::{Iterable, Options};
use crate::err::Error;
use crate::idx::planner::executor::QueryExecutor;
use crate::idx::planner::plan::{Plan, PlanBuilder};
use crate::idx::planner::tree::{Node, Tree};
use crate::sql::{Cond, Operator, Table};
use std::collections::HashMap;
pub(crate) struct QueryPlanner<'a> {
opt: &'a Options,
cond: &'a Option<Cond>,
executors: HashMap<String, QueryExecutor>,
}
impl<'a> QueryPlanner<'a> {
pub(crate) fn new(opt: &'a Options, cond: &'a Option<Cond>) -> Self {
Self {
opt,
cond,
executors: HashMap::default(),
}
}
pub(crate) async fn get_iterable(
&mut self,
ctx: &Context<'_>,
opt: &Options,
t: Table,
) -> Result<Iterable, Error> {
let txn = ctx.clone_transaction()?;
let res = Tree::build(self.opt, &txn, &t, self.cond).await?;
if let Some((node, im)) = res {
if let Some(plan) = AllAndStrategy::build(&node)? {
let e = plan.i.new_query_executor(opt, &txn, &t, im).await?;
self.executors.insert(t.0.clone(), e);
return Ok(Iterable::Index(t, plan));
}
let e = QueryExecutor::new(opt, &txn, &t, im, None).await?;
self.executors.insert(t.0.clone(), e);
}
Ok(Iterable::Table(t))
}
pub(crate) fn finish(self) -> Option<HashMap<String, QueryExecutor>> {
if self.executors.is_empty() {
None
} else {
Some(self.executors)
}
}
}
struct AllAndStrategy {
b: PlanBuilder,
}
/// Successful if every boolean operators are AND
/// and there is at least one condition covered by an index
impl AllAndStrategy {
fn build(node: &Node) -> Result<Option<Plan>, Error> {
let mut s = AllAndStrategy {
b: PlanBuilder::default(),
};
match s.eval_node(node) {
Ok(_) => match s.b.build() {
Ok(p) => Ok(Some(p)),
Err(Error::BypassQueryPlanner) => Ok(None),
Err(e) => Err(e),
},
Err(Error::BypassQueryPlanner) => Ok(None),
Err(e) => Err(e),
}
}
fn eval_node(&mut self, node: &Node) -> Result<(), Error> {
match node {
Node::Expression {
index_option,
left,
right,
operator,
} => {
if let Some(io) = index_option {
self.b.add(io.clone());
}
self.eval_expression(left, right, operator)
}
Node::Unsupported => Err(Error::BypassQueryPlanner),
_ => Ok(()),
}
}
fn eval_expression(&mut self, left: &Node, right: &Node, op: &Operator) -> Result<(), Error> {
if op.eq(&Operator::Or) {
return Err(Error::BypassQueryPlanner);
}
self.eval_node(left)?;
self.eval_node(right)?;
Ok(())
}
}

277
lib/src/idx/planner/plan.rs Normal file
View file

@ -0,0 +1,277 @@
use crate::dbs::{Options, Transaction};
use crate::err::Error;
use crate::idx::ft::{FtIndex, HitsIterator};
use crate::idx::planner::executor::QueryExecutor;
use crate::idx::planner::tree::{IndexMap, Node};
use crate::idx::IndexKeyBase;
use crate::key;
use crate::kvs::Key;
use crate::sql::index::Index;
use crate::sql::scoring::Scoring;
use crate::sql::statements::DefineIndexStatement;
use crate::sql::{Array, Expression, Ident, Object, Operator, Table, Thing, Value};
use async_trait::async_trait;
use std::collections::HashMap;
#[derive(Default)]
pub(super) struct PlanBuilder {
indexes: Vec<IndexOption>,
}
impl PlanBuilder {
pub(super) fn add(&mut self, i: IndexOption) {
self.indexes.push(i);
}
pub(super) fn build(mut self) -> Result<Plan, Error> {
// TODO select the best option if there are several (cost based)
if let Some(index) = self.indexes.pop() {
Ok(index.into())
} else {
Err(Error::BypassQueryPlanner)
}
}
}
pub(crate) struct Plan {
pub(super) i: IndexOption,
}
impl Plan {
pub(crate) async fn new_iterator(
&self,
opt: &Options,
txn: &Transaction,
) -> Result<Box<dyn ThingIterator>, Error> {
self.i.new_iterator(opt, txn).await
}
pub(crate) fn explain(&self) -> Value {
match &self.i {
IndexOption {
ix,
v,
op,
..
} => Value::Object(Object::from(HashMap::from([
("index", Value::from(ix.name.0.to_owned())),
("operator", Value::from(op.to_string())),
("value", v.clone()),
]))),
}
}
}
impl From<IndexOption> for Plan {
fn from(i: IndexOption) -> Self {
Self {
i,
}
}
}
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub(super) struct IndexOption {
pub(super) ix: DefineIndexStatement,
pub(super) v: Value,
pub(super) op: Operator,
ep: Expression,
}
impl IndexOption {
fn new(ix: DefineIndexStatement, op: Operator, v: Value, ep: Expression) -> Self {
Self {
ix,
op,
v,
ep,
}
}
pub(super) async fn new_query_executor(
&self,
opt: &Options,
txn: &Transaction,
t: &Table,
i: IndexMap,
) -> Result<QueryExecutor, Error> {
QueryExecutor::new(opt, txn, t, i, Some(self.ep.clone())).await
}
pub(super) fn found(
ix: &DefineIndexStatement,
op: &Operator,
v: &Node,
ep: &Expression,
) -> Option<Self> {
if let Some(v) = v.is_scalar() {
if match ix.index {
Index::Idx => Operator::Equal.eq(op),
Index::Uniq => Operator::Equal.eq(op),
Index::Search {
..
} => {
matches!(op, Operator::Matches(_))
}
} {
return Some(IndexOption::new(ix.clone(), op.to_owned(), v.clone(), ep.clone()));
}
}
None
}
async fn new_iterator(
&self,
opt: &Options,
txn: &Transaction,
) -> Result<Box<dyn ThingIterator>, Error> {
match &self.ix.index {
Index::Idx => match self.op {
Operator::Equal => {
Ok(Box::new(NonUniqueEqualThingIterator::new(opt, &self.ix, &self.v)?))
}
_ => Err(Error::BypassQueryPlanner),
},
Index::Uniq => match self.op {
Operator::Equal => {
Ok(Box::new(UniqueEqualThingIterator::new(opt, &self.ix, &self.v)?))
}
_ => Err(Error::BypassQueryPlanner),
},
Index::Search {
az,
hl,
sc,
order,
} => match self.op {
Operator::Matches(_) => Ok(Box::new(
MatchesThingIterator::new(opt, txn, &self.ix, az, *hl, sc, *order, &self.v)
.await?,
)),
_ => Err(Error::BypassQueryPlanner),
},
}
}
}
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
pub(crate) trait ThingIterator: Send {
async fn next_batch(&mut self, tx: &Transaction, size: u32) -> Result<Vec<Thing>, Error>;
}
struct NonUniqueEqualThingIterator {
beg: Vec<u8>,
end: Vec<u8>,
}
impl NonUniqueEqualThingIterator {
fn new(opt: &Options, ix: &DefineIndexStatement, v: &Value) -> Result<Self, Error> {
let v = Array::from(v.clone());
let beg = key::index::prefix_all_ids(opt.ns(), opt.db(), &ix.what, &ix.name, &v);
let end = key::index::suffix_all_ids(opt.ns(), opt.db(), &ix.what, &ix.name, &v);
Ok(Self {
beg,
end,
})
}
}
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
impl ThingIterator for NonUniqueEqualThingIterator {
async fn next_batch(&mut self, txn: &Transaction, limit: u32) -> Result<Vec<Thing>, Error> {
let min = self.beg.clone();
let max = self.end.clone();
let res = txn.lock().await.scan(min..max, limit).await?;
if let Some((key, _)) = res.last() {
self.beg = key.clone();
self.beg.push(0x00);
}
let res = res.iter().map(|(_, val)| val.into()).collect();
Ok(res)
}
}
struct UniqueEqualThingIterator {
key: Option<Key>,
}
impl UniqueEqualThingIterator {
fn new(opt: &Options, ix: &DefineIndexStatement, v: &Value) -> Result<Self, Error> {
let v = Array::from(v.clone());
let key = key::index::new(opt.ns(), opt.db(), &ix.what, &ix.name, &v, None).into();
Ok(Self {
key: Some(key),
})
}
}
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
impl ThingIterator for UniqueEqualThingIterator {
async fn next_batch(&mut self, txn: &Transaction, _limit: u32) -> Result<Vec<Thing>, Error> {
if let Some(key) = self.key.take() {
if let Some(val) = txn.lock().await.get(key).await? {
return Ok(vec![val.into()]);
}
}
Ok(vec![])
}
}
struct MatchesThingIterator {
hits: Option<HitsIterator>,
}
impl MatchesThingIterator {
async fn new(
opt: &Options,
txn: &Transaction,
ix: &DefineIndexStatement,
az: &Ident,
_hl: bool,
sc: &Scoring,
order: u32,
v: &Value,
) -> Result<Self, Error> {
let ikb = IndexKeyBase::new(opt, ix);
let mut run = txn.lock().await;
if let Scoring::Bm {
..
} = sc
{
let query_string = v.clone().convert_to_string()?;
let az = run.get_az(opt.ns(), opt.db(), az.as_str()).await?;
let fti = FtIndex::new(&mut run, az, ikb, order).await?;
let hits = fti.search(&mut run, query_string).await?;
Ok(Self {
hits,
})
} else {
Err(Error::FeatureNotYetImplemented {
feature: "Vector Search",
})
}
}
}
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
impl ThingIterator for MatchesThingIterator {
async fn next_batch(&mut self, txn: &Transaction, mut limit: u32) -> Result<Vec<Thing>, Error> {
let mut res = vec![];
if let Some(hits) = &mut self.hits {
let mut run = txn.lock().await;
while limit > 0 {
if let Some((hit, _)) = hits.next(&mut run).await? {
res.push(hit);
} else {
break;
}
limit -= 1;
}
}
Ok(res)
}
}

162
lib/src/idx/planner/tree.rs Normal file
View file

@ -0,0 +1,162 @@
use crate::dbs::{Options, Transaction};
use crate::err::Error;
use crate::idx::planner::plan::IndexOption;
use crate::sql::statements::DefineIndexStatement;
use crate::sql::{Cond, Expression, Idiom, Operator, Subquery, Table, Value};
use async_recursion::async_recursion;
use std::collections::hash_map::Entry;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
pub(super) struct Tree {}
pub(super) type IndexMap = HashMap<Expression, HashSet<IndexOption>>;
impl Tree {
pub(super) async fn build<'a>(
opt: &'a Options,
txn: &'a Transaction,
table: &'a Table,
cond: &Option<Cond>,
) -> Result<Option<(Node, IndexMap)>, Error> {
let mut b = TreeBuilder {
opt,
txn,
table,
indexes: None,
index_map: IndexMap::default(),
};
let mut res = None;
if let Some(cond) = cond {
res = Some((b.eval_value(&cond.0).await?, b.index_map));
}
Ok(res)
}
}
struct TreeBuilder<'a> {
opt: &'a Options,
txn: &'a Transaction,
table: &'a Table,
indexes: Option<Arc<[DefineIndexStatement]>>,
index_map: IndexMap,
}
impl<'a> TreeBuilder<'a> {
async fn find_index(&mut self, i: &Idiom) -> Result<Option<DefineIndexStatement>, Error> {
if self.indexes.is_none() {
let indexes = self
.txn
.clone()
.lock()
.await
.all_ix(self.opt.ns(), self.opt.db(), &self.table.0)
.await?;
self.indexes = Some(indexes);
}
if let Some(indexes) = &self.indexes {
for ix in indexes.as_ref() {
if ix.cols.len() == 1 && ix.cols[0].eq(i) {
return Ok(Some(ix.clone()));
}
}
}
Ok(None)
}
#[cfg_attr(not(target_arch = "wasm32"), async_recursion)]
#[cfg_attr(target_arch = "wasm32", async_recursion(?Send))]
async fn eval_value(&mut self, v: &Value) -> Result<Node, Error> {
Ok(match v {
Value::Expression(e) => self.eval_expression(e).await?,
Value::Idiom(i) => self.eval_idiom(i).await?,
Value::Strand(_) => Node::Scalar(v.to_owned()),
Value::Number(_) => Node::Scalar(v.to_owned()),
Value::Bool(_) => Node::Scalar(v.to_owned()),
Value::Subquery(s) => self.eval_subquery(s).await?,
_ => Node::Unsupported,
})
}
async fn eval_idiom(&mut self, i: &Idiom) -> Result<Node, Error> {
Ok(if let Some(ix) = self.find_index(i).await? {
Node::IndexedField(ix)
} else {
Node::NonIndexedField
})
}
async fn eval_expression(&mut self, e: &Expression) -> Result<Node, Error> {
let left = self.eval_value(&e.l).await?;
let right = self.eval_value(&e.r).await?;
let mut index_option = None;
if let Some(ix) = left.is_indexed_field() {
if let Some(io) = IndexOption::found(ix, &e.o, &right, e) {
index_option = Some(io.clone());
self.add_index(e, io);
}
}
if let Some(ix) = right.is_indexed_field() {
if let Some(io) = IndexOption::found(ix, &e.o, &left, e) {
index_option = Some(io.clone());
self.add_index(e, io);
}
}
Ok(Node::Expression {
index_option,
left: Box::new(left),
right: Box::new(right),
operator: e.o.to_owned(),
})
}
fn add_index(&mut self, e: &Expression, io: IndexOption) {
match self.index_map.entry(e.clone()) {
Entry::Occupied(mut e) => {
e.get_mut().insert(io);
}
Entry::Vacant(e) => {
e.insert(HashSet::from([io]));
}
}
}
async fn eval_subquery(&mut self, s: &Subquery) -> Result<Node, Error> {
Ok(match s {
Subquery::Value(v) => self.eval_value(v).await?,
_ => Node::Unsupported,
})
}
}
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub(super) enum Node {
Expression {
index_option: Option<IndexOption>,
left: Box<Node>,
right: Box<Node>,
operator: Operator,
},
IndexedField(DefineIndexStatement),
NonIndexedField,
Scalar(Value),
Unsupported,
}
impl Node {
pub(super) fn is_scalar(&self) -> Option<&Value> {
if let Node::Scalar(v) = self {
Some(v)
} else {
None
}
}
pub(super) fn is_indexed_field(&self) -> Option<&DefineIndexStatement> {
if let Node::IndexedField(ix) = self {
Some(ix)
} else {
None
}
}
}

59
lib/src/key/bc.rs Normal file
View file

@ -0,0 +1,59 @@
use crate::idx::ft::terms::TermId;
use derive::Key;
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Serialize, Deserialize, Key)]
pub struct Bc<'a> {
__: u8,
_a: u8,
pub ns: &'a str,
_b: u8,
pub db: &'a str,
_c: u8,
pub tb: &'a str,
_d: u8,
_e: u8,
_f: u8,
pub ix: &'a str,
_g: u8,
pub term_id: TermId,
}
impl<'a> Bc<'a> {
pub fn new(ns: &'a str, db: &'a str, tb: &'a str, ix: &'a str, term_id: TermId) -> Self {
Self {
__: b'/',
_a: b'*',
ns,
_b: b'*',
db,
_c: b'*',
tb,
_d: b'!',
_e: b'b',
_f: b'c',
ix,
_g: b'*',
term_id,
}
}
}
#[cfg(test)]
mod tests {
#[test]
fn key() {
use super::*;
#[rustfmt::skip]
let val = Bc::new(
"test",
"test",
"test",
"test",
7
);
let enc = Bc::encode(&val).unwrap();
let dec = Bc::decode(&enc).unwrap();
assert_eq!(val, dec);
}
}

16
lib/src/key/bd.rs
View file

@ -28,18 +28,18 @@ impl<'a> Bd<'a> {
node_id: Option<NodeId>,
) -> Self {
Self {
__: 0x2f, // /
_a: 0x2a, // *
__: b'/',
_a: b'*',
ns,
_b: 0x2a, // *
_b: b'*',
db,
_c: 0x2a, // *
_c: b'*',
tb,
_d: 0x21, // !
_e: 0x62, // b
_f: 0x64, // d
_d: b'!',
_e: b'b',
_f: b'd',
ix,
_g: 0x2a, // *
_g: b'*',
node_id,
}
}

72
lib/src/key/bf.rs
View file

@ -31,66 +31,30 @@ impl<'a> Bf<'a> {
doc_id: DocId,
) -> Self {
Self {
__: 0x2f, // /
_a: 0x2a, // *
__: b'/',
_a: b'*',
ns,
_b: 0x2a, // *
_b: b'*',
db,
_c: 0x2a, // *
_c: b'*',
tb,
_d: 0x21, // !
_e: 0x62, // b
_f: 0x78, // x
_d: b'!',
_e: b'b',
_f: b'f',
ix,
_g: 0x2a, // *
_g: b'*',
term_id,
doc_id,
}
}
}
#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Serialize, Deserialize, Key)]
pub struct BfPrefix<'a> {
__: u8,
_a: u8,
pub ns: &'a str,
_b: u8,
pub db: &'a str,
_c: u8,
pub tb: &'a str,
_d: u8,
_e: u8,
_f: u8,
pub ix: &'a str,
_g: u8,
term_id: TermId,
}
impl<'a> BfPrefix<'a> {
pub fn new(ns: &'a str, db: &'a str, tb: &'a str, ix: &'a str, term_id: TermId) -> Self {
Self {
__: 0x2f, // /
_a: 0x2a, // *
ns,
_b: 0x2a, // *
db,
_c: 0x2a, // *
tb,
_d: 0x21, // !
_e: 0x62, // b
_f: 0x78, // x
ix,
_g: 0x2a, // *
term_id,
}
}
}
#[cfg(test)]
mod tests {
use crate::key::bf::Bf;
#[test]
fn key() {
use super::*;
#[rustfmt::skip]
let val = Bf::new(
"test",
@ -104,20 +68,4 @@ mod tests {
let dec = Bf::decode(&enc).unwrap();
assert_eq!(val, dec);
}
#[test]
fn key_prefix() {
use super::*;
#[rustfmt::skip]
let val = BfPrefix::new(
"test",
"test",
"test",
"test",
3
);
let enc = BfPrefix::encode(&val).unwrap();
let dec = BfPrefix::decode(&enc).unwrap();
assert_eq!(val, dec);
}
}

16
lib/src/key/bi.rs
View file

@ -22,18 +22,18 @@ pub struct Bi<'a> {
impl<'a> Bi<'a> {
pub fn new(ns: &'a str, db: &'a str, tb: &'a str, ix: &'a str, node_id: NodeId) -> Self {
Bi {
__: 0x2f, // /
_a: 0x2a, // *
__: b'/',
_a: b'*',
ns,
_b: 0x2a, // *
_b: b'*',
db,
_c: 0x2a, // *
_c: b'*',
tb,
_d: 0x21, // !
_e: 0x62, // b
_f: 0x69, // i
_d: b'!',
_e: b'b',
_f: b'i',
ix,
_g: 0x2a, // *
_g: b'*',
node_id,
}
}

16
lib/src/key/bk.rs
View file

@ -22,18 +22,18 @@ pub struct Bk<'a> {
impl<'a> Bk<'a> {
pub fn new(ns: &'a str, db: &'a str, tb: &'a str, ix: &'a str, doc_id: DocId) -> Self {
Self {
__: 0x2f, // /
_a: 0x2a, // *
__: b'/',
_a: b'*',
ns,
_b: 0x2a, // *
_b: b'*',
db,
_c: 0x2a, // *
_c: b'*',
tb,
_d: 0x21, // !
_e: 0x62, // b
_f: 0x6b, // k
_d: b'!',
_e: b'b',
_f: b'k',
ix,
_g: 0x2a, // *
_g: b'*',
doc_id,
}
}

16
lib/src/key/bl.rs
View file

@ -28,18 +28,18 @@ impl<'a> Bl<'a> {
node_id: Option<NodeId>,
) -> Self {
Self {
__: 0x2f, // /
_a: 0x2a, // *
__: b'/',
_a: b'*',
ns,
_b: 0x2a, // *
_b: b'*',
db,
_c: 0x2a, // *
_c: b'*',
tb,
_d: 0x21, // !
_e: 0x62, // b
_f: 0x6c, // l
_d: b'!',
_e: b'b',
_f: b'l',
ix,
_g: 0x2a, // *
_g: b'*',
node_id,
}
}

16
lib/src/key/bp.rs
View file

@ -28,18 +28,18 @@ impl<'a> Bp<'a> {
node_id: Option<NodeId>,
) -> Self {
Self {
__: 0x2f, // /
_a: 0x2a, // *
__: b'/',
_a: b'*',
ns,
_b: 0x2a, // *
_b: b'*',
db,
_c: 0x2a, // *
_c: b'*',
tb,
_d: 0x21, // !
_e: 0x62, // b
_f: 0x70, // p
_d: b'!',
_e: b'b',
_f: b'p',
ix,
_g: 0x2a, // *
_g: b'*',
node_id,
}
}

14
lib/src/key/bs.rs
View file

@ -19,16 +19,16 @@ pub struct Bs<'a> {
impl<'a> Bs<'a> {
pub fn new(ns: &'a str, db: &'a str, tb: &'a str, ix: &'a str) -> Self {
Bs {
__: 0x2f, // /
_a: 0x2a, // *
__: b'/',
_a: b'*',
ns,
_b: 0x2a, // *
_b: b'*',
db,
_c: 0x2a, // *
_c: b'*',
tb,
_d: 0x21, // !
_e: 0x62, // b
_f: 0x73, // s,
_d: b'!',
_e: b'b',
_f: b's',
ix,
}
}

16
lib/src/key/bt.rs
View file

@ -28,18 +28,18 @@ impl<'a> Bt<'a> {
node_id: Option<NodeId>,
) -> Self {
Self {
__: 0x2f, // /
_a: 0x2a, // *
__: b'/',
_a: b'*',
ns,
_b: 0x2a, // *
_b: b'*',
db,
_c: 0x2a, // *
_c: b'*',
tb,
_d: 0x21, // !
_e: 0x62, // b
_f: 0x74, // t
_d: b'!',
_e: b'b',
_f: b't',
ix,
_g: 0x2a, // *
_g: b'*',
node_id,
}
}

16
lib/src/key/bu.rs
View file

@ -22,18 +22,18 @@ pub struct Bu<'a> {
impl<'a> Bu<'a> {
pub fn new(ns: &'a str, db: &'a str, tb: &'a str, ix: &'a str, term_id: TermId) -> Self {
Self {
__: 0x2f, // /
_a: 0x2a, // *
__: b'/',
_a: b'*',
ns,
_b: 0x2a, // *
_b: b'*',
db,
_c: 0x2a, // *
_c: b'*',
tb,
_d: 0x21, // !
_e: 0x62, // b
_f: 0x75, // u
_d: b'!',
_e: b'b',
_f: b'u',
ix,
_g: 0x2a, // *
_g: b'*',
term_id,
}
}

53
lib/src/key/index.rs
View file

@ -15,6 +15,7 @@ struct Prefix<'a> {
pub tb: &'a str,
_d: u8,
pub ix: &'a str,
_e: u8,
}
impl<'a> Prefix<'a> {
@ -29,6 +30,42 @@ impl<'a> Prefix<'a> {
tb,
_d: CHAR_INDEX,
ix,
_e: b'*',
}
}
}
#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Serialize, Deserialize, Key)]
struct PrefixIds<'a> {
__: u8,
_a: u8,
pub ns: &'a str,
_b: u8,
pub db: &'a str,
_c: u8,
pub tb: &'a str,
_d: u8,
pub ix: &'a str,
_e: u8,
pub fd: Array,
_f: u8,
}
impl<'a> PrefixIds<'a> {
fn new(ns: &'a str, db: &'a str, tb: &'a str, ix: &'a str, fd: &Array) -> Self {
Self {
__: b'/',
_a: b'*',
ns,
_b: b'*',
db,
_c: b'*',
tb,
_d: CHAR_INDEX,
ix,
_e: b'*',
fd: fd.to_owned(),
_f: b'*',
}
}
}
@ -44,7 +81,9 @@ pub struct Index<'a> {
pub tb: &'a str,
_d: u8,
pub ix: &'a str,
_e: u8,
pub fd: Array,
_f: u8,
pub id: Option<Id>,
}
@ -71,6 +110,18 @@ pub fn suffix(ns: &str, db: &str, tb: &str, ix: &str) -> Vec<u8> {
k
}
pub fn prefix_all_ids(ns: &str, db: &str, tb: &str, ix: &str, fd: &Array) -> Vec<u8> {
let mut k = PrefixIds::new(ns, db, tb, ix, fd).encode().unwrap();
k.extend_from_slice(&[0x00]);
k
}
pub fn suffix_all_ids(ns: &str, db: &str, tb: &str, ix: &str, fd: &Array) -> Vec<u8> {
let mut k = PrefixIds::new(ns, db, tb, ix, fd).encode().unwrap();
k.extend_from_slice(&[0xff]);
k
}
impl<'a> Index<'a> {
pub fn new(
ns: &'a str,
@ -90,7 +141,9 @@ impl<'a> Index<'a> {
tb,
_d: CHAR_INDEX,
ix,
_e: 0x2a, // *
fd,
_f: 0x2a, // *
id,
}
}

15
lib/src/key/mod.rs
View file

@ -34,11 +34,18 @@
///
/// Index /*{ns}*{db}*{tb}¤{ix}{fd}{id}
///
/// BD /*{ns}*{db}*{tb}¤{ix}{bd}{id}
/// BL /*{ns}*{db}*{tb}¤{ix}{bl}{id}
/// BP /*{ns}*{db}*{tb}¤{ix}{bp}{id}
/// BT /*{ns}*{db}*{tb}¤{ix}{bt}{id}
/// BC /*{ns}*{db}*{tb}!bc{ix}*{id}
/// BD /*{ns}*{db}*{tb}!bd{ix}*{id}
/// BF /*{ns}*{db}*{tb}!bf{ix}*{id}
/// BI /*{ns}*{db}*{tb}!bi{ix}*{id}
/// BK /*{ns}*{db}*{tb}!bk{ix}*{id}
/// BL /*{ns}*{db}*{tb}!bl{ix}*{id}
/// BP /*{ns}*{db}*{tb}!bp{ix}*{id}
/// BS /*{ns}*{db}*{tb}!bs{ix}
/// BT /*{ns}*{db}*{tb}!bt{ix}*{id}
/// BU /*{ns}*{db}*{tb}!bu{ix}*{id}
pub mod az; // Stores a DEFINE ANALYZER config definition
pub mod bc; // Stores Doc list for each term
pub mod bd; // Stores BTree nodes for doc ids
pub mod bf; // Stores Term/Doc frequency
pub mod bi; // Stores doc keys for doc_ids

4
lib/src/kvs/ds.rs
View file

@ -413,6 +413,8 @@ impl Datastore {
let mut opt = Options::default();
// Create a default context
let mut ctx = Context::default();
// Add the transaction
ctx.add_transaction(Some(&txn));
// Set the global query timeout
if let Some(timeout) = self.query_timeout {
ctx.add_timeout(timeout);
@ -429,7 +431,7 @@ impl Datastore {
// Set strict config
opt.strict = strict;
// Compute the value
let res = val.compute(&ctx, &opt, &txn, None).await?;
let res = val.compute(&ctx, &opt).await?;
// Store any data
match val.writeable() {
true => txn.lock().await.commit().await?,

14
lib/src/kvs/tx.rs
View file

@ -1316,6 +1316,20 @@ impl Transaction {
})?;
Ok(val.into())
}
/// Retrieve a specific analyzer definition.
pub async fn get_ix(
&mut self,
ns: &str,
db: &str,
tb: &str,
ix: &str,
) -> Result<DefineIndexStatement, Error> {
let key = crate::key::ix::new(ns, db, tb, ix);
let val = self.get(key).await?.ok_or(Error::IxNotFound {
value: ix.to_owned(),
})?;
Ok(val.into())
}
/// Add a namespace with a default configuration, only if we are in dynamic mode.
pub async fn add_ns(
&mut self,

11
lib/src/sql/array.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::mightbespace;
use crate::sql::common::{closebracket, commas, openbracket};
@ -112,16 +111,10 @@ impl Array {
impl Array {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
let mut x = Self::with_capacity(self.len());
for v in self.iter() {
match v.compute(ctx, opt, txn, doc).await {
match v.compute(ctx, opt).await {
Ok(v) => x.push(v),
Err(e) => return Err(e),
};

29
lib/src/sql/block.rs
View file

@ -1,7 +1,6 @@
use crate::cnf::PROTECTED_PARAM_NAMES;
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::{comment, mightbespace};
use crate::sql::common::{closebraces, colons, openbraces};
@ -52,13 +51,7 @@ impl Block {
self.iter().any(Entry::writeable)
}
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Duplicate context
let mut ctx = Context::new(ctx);
// Loop over the statements
@ -68,7 +61,7 @@ impl Block {
// Check if the variable is a protected variable
let val = match PROTECTED_PARAM_NAMES.contains(&v.name.as_str()) {
// The variable isn't protected and can be stored
false => v.compute(&ctx, opt, txn, doc).await,
false => v.compute(&ctx, opt).await,
// The user tried to set a protected variable
true => {
return Err(Error::InvalidParam {
@ -80,31 +73,31 @@ impl Block {
ctx.add_value(v.name.to_owned(), val);
}
Entry::Ifelse(v) => {
v.compute(&ctx, opt, txn, doc).await?;
v.compute(&ctx, opt).await?;
}
Entry::Select(v) => {
v.compute(&ctx, opt, txn, doc).await?;
v.compute(&ctx, opt).await?;
}
Entry::Create(v) => {
v.compute(&ctx, opt, txn, doc).await?;
v.compute(&ctx, opt).await?;
}
Entry::Update(v) => {
v.compute(&ctx, opt, txn, doc).await?;
v.compute(&ctx, opt).await?;
}
Entry::Delete(v) => {
v.compute(&ctx, opt, txn, doc).await?;
v.compute(&ctx, opt).await?;
}
Entry::Relate(v) => {
v.compute(&ctx, opt, txn, doc).await?;
v.compute(&ctx, opt).await?;
}
Entry::Insert(v) => {
v.compute(&ctx, opt, txn, doc).await?;
v.compute(&ctx, opt).await?;
}
Entry::Output(v) => {
return v.compute(&ctx, opt, txn, doc).await;
return v.compute(&ctx, opt).await;
}
Entry::Value(v) => {
return v.compute(&ctx, opt, txn, doc).await;
return v.compute(&ctx, opt).await;
}
}
}

11
lib/src/sql/cast.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::mightbespace;
use crate::sql::error::IResult;
@ -35,17 +34,11 @@ impl Cast {
impl Cast {
#[cfg_attr(not(target_arch = "wasm32"), async_recursion)]
#[cfg_attr(target_arch = "wasm32", async_recursion(?Send))]
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&'async_recursion Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Prevent long cast chains
let opt = &opt.dive(1)?;
// Compute the value to be cast and convert it
self.1.compute(ctx, opt, txn, doc).await?.convert_to(&self.0)
self.1.compute(ctx, opt).await?.convert_to(&self.0)
}
}

9
lib/src/sql/constant.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::error::IResult;
use crate::sql::value::Value;
@ -79,13 +78,7 @@ impl Constant {
}
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
_ctx: &Context<'_>,
_opt: &Options,
_txn: &Transaction,
_doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, _ctx: &Context<'_>, _opt: &Options) -> Result<Value, Error> {
Ok(match self.value() {
ConstantValue::Datetime(d) => d.into(),
ConstantValue::Float(f) => f.into(),

10
lib/src/sql/data.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::mightbespace;
use crate::sql::comment::shouldbespace;
@ -44,26 +43,25 @@ impl Data {
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
tb: &Table,
) -> Result<Thing, Error> {
match self {
Self::MergeExpression(v) => {
// This MERGE expression has an 'id' field
v.compute(ctx, opt, txn, None).await?.rid().generate(tb, false)
v.compute(ctx, opt).await?.rid().generate(tb, false)
}
Self::ReplaceExpression(v) => {
// This REPLACE expression has an 'id' field
v.compute(ctx, opt, txn, None).await?.rid().generate(tb, false)
v.compute(ctx, opt).await?.rid().generate(tb, false)
}
Self::ContentExpression(v) => {
// This CONTENT expression has an 'id' field
v.compute(ctx, opt, txn, None).await?.rid().generate(tb, false)
v.compute(ctx, opt).await?.rid().generate(tb, false)
}
Self::SetExpression(v) => match v.iter().find(|f| f.0.is_id()) {
Some((_, _, v)) => {
// This SET expression has an 'id' field
v.compute(ctx, opt, txn, None).await?.generate(tb, false)
v.compute(ctx, opt).await?.generate(tb, false)
}
// This SET expression had no 'id' field
_ => Ok(tb.generate()),

16
lib/src/sql/expression.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::fnc;
use crate::sql::error::IResult;
@ -32,7 +31,7 @@ impl Default for Expression {
impl Expression {
/// Create a new expression
fn new(l: Value, o: Operator, r: Value) -> Self {
pub(crate) fn new(l: Value, o: Operator, r: Value) -> Self {
Self {
l,
o,
@ -61,14 +60,8 @@ impl Expression {
impl Expression {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
let l = self.l.compute(ctx, opt, txn, doc).await?;
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
let l = self.l.compute(ctx, opt).await?;
match self.o {
Operator::Or => {
if let true = l.is_truthy() {
@ -92,7 +85,7 @@ impl Expression {
}
_ => {} // Continue
}
let r = self.r.compute(ctx, opt, txn, doc).await?;
let r = self.r.compute(ctx, opt).await?;
match self.o {
Operator::Or => fnc::operate::or(l, r),
Operator::And => fnc::operate::and(l, r),
@ -128,6 +121,7 @@ impl Expression {
Operator::NoneInside => fnc::operate::inside_none(&l, &r),
Operator::Outside => fnc::operate::outside(&l, &r),
Operator::Intersects => fnc::operate::intersects(&l, &r),
Operator::Matches(_) => fnc::operate::matches(ctx, self).await,
_ => unreachable!(),
}
}

34
lib/src/sql/field.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::shouldbespace;
use crate::sql::common::commas;
@ -76,17 +75,17 @@ impl Fields {
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
group: bool,
) -> Result<Value, Error> {
// Ensure futures are run
let opt = &opt.futures(true);
//
let doc = doc.unwrap_or(&Value::None);
let doc = ctx.doc().unwrap_or(&Value::None);
let mut ctx = Context::new(ctx);
ctx.add_cursor_doc(doc);
// Process the desired output
let mut out = match self.is_all() {
true => doc.compute(ctx, opt, txn, Some(doc)).await?,
true => doc.compute(&ctx, opt).await?,
false => Value::base(),
};
for v in self.other() {
@ -105,13 +104,13 @@ impl Fields {
Value::Function(f) if group && f.is_aggregate() => {
let x = match f.args().len() {
// If no function arguments, then compute the result
0 => f.compute(ctx, opt, txn, Some(doc)).await?,
0 => f.compute(&ctx, opt).await?,
// If arguments, then pass the first value through
_ => f.args()[0].compute(ctx, opt, txn, Some(doc)).await?,
_ => f.args()[0].compute(&ctx, opt).await?,
};
// Check if this is a single VALUE field expression
match self.single().is_some() {
false => out.set(ctx, opt, txn, idiom.as_ref(), x).await?,
false => out.set(&ctx, opt, idiom.as_ref(), x).await?,
true => out = x,
}
}
@ -127,12 +126,8 @@ impl Fields {
None => doc,
};
// Continue fetching the next idiom part
let x = x
.get(ctx, opt, txn, Some(doc), v)
.await?
.compute(ctx, opt, txn, Some(doc))
.await?
.flatten();
let x =
x.get(&ctx, opt, v).await?.compute(&ctx, opt).await?.flatten();
// Add the result to the temporary store
res.push((v, x));
}
@ -142,24 +137,23 @@ impl Fields {
// This is an alias expression part
Some(a) => {
if let Some(i) = alias {
out.set(ctx, opt, txn, i, x.clone()).await?;
out.set(&ctx, opt, i, x.clone()).await?;
}
out.set(ctx, opt, txn, a, x).await?;
out.set(&ctx, opt, a, x).await?;
}
// This is the end of the expression
None => {
out.set(ctx, opt, txn, alias.as_ref().unwrap_or(v), x)
.await?
out.set(&ctx, opt, alias.as_ref().unwrap_or(v), x).await?
}
}
}
}
// This expression is a normal field expression
_ => {
let x = expr.compute(ctx, opt, txn, Some(doc)).await?;
let x = expr.compute(&ctx, opt).await?;
// Check if this is a single VALUE field expression
match self.single().is_some() {
false => out.set(ctx, opt, txn, idiom.as_ref(), x).await?,
false => out.set(&ctx, opt, idiom.as_ref(), x).await?,
true => out = x,
}
}

43
lib/src/sql/filter.rs
View file

@ -2,10 +2,9 @@ use crate::sql::comment::shouldbespace;
use crate::sql::common::{closeparentheses, commas, openparentheses};
use crate::sql::error::IResult;
use crate::sql::language::{language, Language};
use crate::sql::number::number;
use crate::sql::Number;
use nom::branch::alt;
use nom::bytes::complete::tag_no_case;
use nom::character::complete::u16;
use nom::multi::separated_list1;
use serde::{Deserialize, Serialize};
use std::fmt;
@ -13,31 +12,57 @@ use std::fmt::Display;
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Hash)]
pub enum Filter {
EdgeNgram(Number, Number),
Ascii,
EdgeNgram(u16, u16),
Lowercase,
Ngram(u16, u16),
Snowball(Language),
Uppercase,
}
impl Display for Filter {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Self::Ascii => f.write_str("ASCII"),
Self::EdgeNgram(min, max) => write!(f, "EDGENGRAM({},{})", min, max),
Self::Lowercase => f.write_str("LOWERCASE"),
Self::Ngram(min, max) => write!(f, "NGRAM({},{})", min, max),
Self::Snowball(lang) => write!(f, "SNOWBALL({})", lang),
Self::Uppercase => f.write_str("UPPERCASE"),
}
}
}
fn ascii(i: &str) -> IResult<&str, Filter> {
let (i, _) = tag_no_case("ASCII")(i)?;
Ok((i, Filter::Ascii))
}
fn edgengram(i: &str) -> IResult<&str, Filter> {
let (i, _) = tag_no_case("EDGENGRAM")(i)?;
let (i, _) = openparentheses(i)?;
let (i, min) = number(i)?;
let (i, min) = u16(i)?;
let (i, _) = commas(i)?;
let (i, max) = number(i)?;
let (i, max) = u16(i)?;
let (i, _) = closeparentheses(i)?;
Ok((i, Filter::EdgeNgram(min, max)))
}
fn ngram(i: &str) -> IResult<&str, Filter> {
let (i, _) = tag_no_case("NGRAM")(i)?;
let (i, _) = openparentheses(i)?;
let (i, min) = u16(i)?;
let (i, _) = commas(i)?;
let (i, max) = u16(i)?;
let (i, _) = closeparentheses(i)?;
Ok((i, Filter::Ngram(min, max)))
}
fn lowercase(i: &str) -> IResult<&str, Filter> {
let (i, _) = tag_no_case("LOWERCASE")(i)?;
Ok((i, Filter::Lowercase))
}
fn snowball(i: &str) -> IResult<&str, Filter> {
let (i, _) = tag_no_case("SNOWBALL")(i)?;
let (i, _) = openparentheses(i)?;
@ -46,13 +71,13 @@ fn snowball(i: &str) -> IResult<&str, Filter> {
Ok((i, Filter::Snowball(language)))
}
fn lowercase(i: &str) -> IResult<&str, Filter> {
let (i, _) = tag_no_case("LOWERCASE")(i)?;
Ok((i, Filter::Lowercase))
fn uppercase(i: &str) -> IResult<&str, Filter> {
let (i, _) = tag_no_case("UPPERCASE")(i)?;
Ok((i, Filter::Uppercase))
}
fn filter(i: &str) -> IResult<&str, Filter> {
alt((edgengram, lowercase, snowball))(i)
alt((ascii, edgengram, lowercase, ngram, snowball, uppercase))(i)
}
pub(super) fn filters(i: &str) -> IResult<&str, Vec<Filter>> {

23
lib/src/sql/function.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::fnc;
use crate::sql::comment::mightbespace;
@ -132,13 +131,7 @@ impl Function {
/// Process this type returning a computed simple Value
#[cfg_attr(not(target_arch = "wasm32"), async_recursion)]
#[cfg_attr(target_arch = "wasm32", async_recursion(?Send))]
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&'async_recursion Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Prevent long function chains
let opt = &opt.dive(1)?;
// Ensure futures are run
@ -147,7 +140,7 @@ impl Function {
match self {
Self::Normal(s, x) => {
// Compute the function arguments
let a = try_join_all(x.iter().map(|v| v.compute(ctx, opt, txn, doc))).await?;
let a = try_join_all(x.iter().map(|v| v.compute(ctx, opt))).await?;
// Run the normal function
fnc::run(ctx, s, a).await
}
@ -155,9 +148,9 @@ impl Function {
// Get the function definition
let val = {
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Get the function definition
run.get_fc(opt.ns(), opt.db(), s).await?
};
@ -172,7 +165,7 @@ impl Function {
});
}
// Compute the function arguments
let a = try_join_all(x.iter().map(|v| v.compute(ctx, opt, txn, doc))).await?;
let a = try_join_all(x.iter().map(|v| v.compute(ctx, opt))).await?;
// Duplicate context
let mut ctx = Context::new(ctx);
// Process the function arguments
@ -180,16 +173,16 @@ impl Function {
ctx.add_value(name.to_raw(), val.coerce_to(&kind)?);
}
// Run the custom function
val.block.compute(&ctx, opt, txn, doc).await
val.block.compute(&ctx, opt).await
}
#[allow(unused_variables)]
Self::Script(s, x) => {
#[cfg(feature = "scripting")]
{
// Compute the function arguments
let a = try_join_all(x.iter().map(|v| v.compute(ctx, opt, txn, doc))).await?;
let a = try_join_all(x.iter().map(|v| v.compute(ctx, opt))).await?;
// Run the script function
fnc::script::run(ctx, opt, txn, doc, s, a).await
fnc::script::run(ctx, opt, s, a).await
}
#[cfg(not(feature = "scripting"))]
{

11
lib/src/sql/future.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::block::{block, Block};
use crate::sql::comment::mightbespace;
@ -25,18 +24,12 @@ impl From<Value> for Future {
impl Future {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Prevent long future chains
let opt = &opt.dive(1)?;
// Process the future if enabled
match opt.futures {
true => self.0.compute(ctx, opt, txn, doc).await?.ok(),
true => self.0.compute(ctx, opt).await?.ok(),
false => Ok(self.clone().into()),
}
}

13
lib/src/sql/id.rs
View file

@ -1,7 +1,6 @@
use crate::cnf::ID_CHARS;
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::array::{array, Array};
use crate::sql::error::IResult;
@ -167,21 +166,15 @@ impl Display for Id {
impl Id {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Id, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Id, Error> {
match self {
Id::Number(v) => Ok(Id::Number(*v)),
Id::String(v) => Ok(Id::String(v.clone())),
Id::Object(v) => match v.compute(ctx, opt, txn, doc).await? {
Id::Object(v) => match v.compute(ctx, opt).await? {
Value::Object(v) => Ok(Id::Object(v)),
_ => unreachable!(),
},
Id::Array(v) => match v.compute(ctx, opt, txn, doc).await? {
Id::Array(v) => match v.compute(ctx, opt).await? {
Value::Array(v) => Ok(Id::Array(v)),
_ => unreachable!(),
},

19
lib/src/sql/idiom.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::common::commas;
use crate::sql::error::IResult;
@ -128,27 +127,21 @@ impl Idiom {
self.0.iter().any(|v| v.writeable())
}
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
match self.first() {
// The starting part is a value
Some(Part::Value(v)) => {
v.compute(ctx, opt, txn, doc)
v.compute(ctx, opt)
.await?
.get(ctx, opt, txn, doc, self.as_ref().next())
.get(ctx, opt, self.as_ref().next())
.await?
.compute(ctx, opt, txn, doc)
.compute(ctx, opt)
.await
}
// Otherwise use the current document
_ => match doc {
_ => match ctx.doc() {
// There is a current document
Some(v) => v.get(ctx, opt, txn, doc, self).await?.compute(ctx, opt, txn, doc).await,
Some(v) => v.get(ctx, opt, self).await?.compute(ctx, opt).await,
// There isn't any document
None => Ok(Value::None),
},

30
lib/src/sql/index.rs
View file

@ -1,10 +1,12 @@
use crate::idx::ft::analyzer::Analyzers;
use crate::sql::comment::{mightbespace, shouldbespace};
use crate::sql::error::IResult;
use crate::sql::ident::{ident, Ident};
use crate::sql::scoring::{scoring, Scoring};
use nom::branch::alt;
use nom::bytes::complete::{tag, tag_no_case};
use nom::combinator::map;
use nom::character::complete::u32;
use nom::combinator::{map, opt};
use serde::{Deserialize, Serialize};
use std::fmt;
@ -19,6 +21,7 @@ pub enum Index {
az: Ident,
hl: bool,
sc: Scoring,
order: u32,
},
}
@ -37,8 +40,9 @@ impl fmt::Display for Index {
az,
hl,
sc,
order,
} => {
write!(f, "SEARCH {} {}", az, sc)?;
write!(f, "SEARCH ANALYZER {} {} ORDER {}", az, sc, order)?;
if *hl {
f.write_str(" HIGHLIGHTS")?
}
@ -62,6 +66,22 @@ pub fn unique(i: &str) -> IResult<&str, Index> {
Ok((i, Index::Uniq))
}
pub fn analyzer(i: &str) -> IResult<&str, Ident> {
let (i, _) = mightbespace(i)?;
let (i, _) = tag_no_case("ANALYZER")(i)?;
let (i, _) = shouldbespace(i)?;
let (i, analyzer) = ident(i)?;
Ok((i, analyzer))
}
pub fn order(i: &str) -> IResult<&str, u32> {
let (i, _) = mightbespace(i)?;
let (i, _) = tag_no_case("ORDER")(i)?;
let (i, _) = shouldbespace(i)?;
let (i, order) = u32(i)?;
Ok((i, order))
}
pub fn highlights(i: &str) -> IResult<&str, bool> {
let (i, _) = mightbespace(i)?;
alt((map(tag("HIGHLIGHTS"), |_| true), map(tag(""), |_| false)))(i)
@ -70,16 +90,18 @@ pub fn highlights(i: &str) -> IResult<&str, bool> {
pub fn search(i: &str) -> IResult<&str, Index> {
let (i, _) = tag_no_case("SEARCH")(i)?;
let (i, _) = shouldbespace(i)?;
let (i, az) = ident(i)?;
let (i, az) = opt(analyzer)(i)?;
let (i, _) = shouldbespace(i)?;
let (i, sc) = scoring(i)?;
let (i, o) = opt(order)(i)?;
let (i, hl) = highlights(i)?;
Ok((
i,
Index::Search {
az,
az: az.unwrap_or_else(|| Ident::from(Analyzers::LIKE)),
sc,
hl,
order: o.unwrap_or(100),
},
))
}

88
lib/src/sql/language.rs
View file

@ -8,17 +8,103 @@ use std::fmt::Display;
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Hash)]
pub enum Language {
Arabic,
Danish,
Dutch,
English,
French,
German,
Greek,
Hungarian,
Italian,
Norwegian,
Portuguese,
Romanian,
Russian,
Spanish,
Swedish,
Tamil,
Turkish,
}
impl Display for Language {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str(match self {
Self::Arabic => "ARABIC",
Self::Danish => "DANISH",
Self::Dutch => "DUTCH",
Self::English => "ENGLISH",
Self::French => "FRENCH",
Self::German => "GERMAN",
Self::Greek => "GREEK",
Self::Hungarian => "HUNGARIAN",
Self::Italian => "ITALIAN",
Self::Norwegian => "NORWEGIAN",
Self::Portuguese => "PORTUGUESE",
Self::Romanian => "ROMANIAN",
Self::Russian => "RUSSIAN",
Self::Spanish => "SPANISH",
Self::Swedish => "SWEDISH",
Self::Tamil => "TAMIL",
Self::Turkish => "TURKISH",
})
}
}
/// Language supports the english name and also ISO 639-1 (3 characters)
/// and ISO 639-2 (2 characters)
pub(super) fn language(i: &str) -> IResult<&str, Language> {
alt((map(tag_no_case("ENGLISH"), |_| Language::English),))(i)
alt((
map(alt((tag_no_case("ARABIC"), tag_no_case("ARA"), tag_no_case("AR"))), |_| {
Language::Arabic
}),
map(alt((tag_no_case("DANISH"), tag_no_case("DAN"), tag_no_case("DA"))), |_| {
Language::Danish
}),
map(alt((tag_no_case("DUTCH"), tag_no_case("NLD"), tag_no_case("NL"))), |_| {
Language::Dutch
}),
map(alt((tag_no_case("ENGLISH"), tag_no_case("ENG"), tag_no_case("EN"))), |_| {
Language::English
}),
map(alt((tag_no_case("FRENCH"), tag_no_case("FRA"), tag_no_case("FR"))), |_| {
Language::French
}),
map(alt((tag_no_case("GERMAN"), tag_no_case("DEU"), tag_no_case("DE"))), |_| {
Language::German
}),
map(alt((tag_no_case("GREEK"), tag_no_case("ELL"), tag_no_case("EL"))), |_| {
Language::Greek
}),
map(alt((tag_no_case("HUNGARIAN"), tag_no_case("HUN"), tag_no_case("HU"))), |_| {
Language::Hungarian
}),
map(alt((tag_no_case("ITALIAN"), tag_no_case("ITA"), tag_no_case("IT"))), |_| {
Language::Italian
}),
map(alt((tag_no_case("NORWEGIAN"), tag_no_case("NOR"), tag_no_case("NO"))), |_| {
Language::Norwegian
}),
map(alt((tag_no_case("PORTUGUESE"), tag_no_case("POR"), tag_no_case("PT"))), |_| {
Language::Portuguese
}),
map(alt((tag_no_case("ROMANIAN"), tag_no_case("RON"), tag_no_case("RO"))), |_| {
Language::Romanian
}),
map(alt((tag_no_case("RUSSIAN"), tag_no_case("RUS"), tag_no_case("RU"))), |_| {
Language::Russian
}),
map(alt((tag_no_case("SPANISH"), tag_no_case("SPA"), tag_no_case("ES"))), |_| {
Language::Spanish
}),
map(alt((tag_no_case("SWEDISH"), tag_no_case("SWE"), tag_no_case("SV"))), |_| {
Language::Swedish
}),
map(alt((tag_no_case("TAMIL"), tag_no_case("TAM"), tag_no_case("TA"))), |_| {
Language::Tamil
}),
map(alt((tag_no_case("TURKISH"), tag_no_case("TUR"), tag_no_case("TR"))), |_| {
Language::Turkish
}),
))(i)
}

11
lib/src/sql/limit.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::shouldbespace;
use crate::sql::error::IResult;
@ -16,14 +15,8 @@ use std::fmt;
pub struct Limit(pub Value);
impl Limit {
pub(crate) async fn process(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<usize, Error> {
match self.0.compute(ctx, opt, txn, doc).await {
pub(crate) async fn process(&self, ctx: &Context<'_>, opt: &Options) -> Result<usize, Error> {
match self.0.compute(ctx, opt).await {
// This is a valid limiting number
Ok(Value::Number(Number::Int(v))) if v >= 0 => Ok(v as usize),
// An invalid value was specified

17
lib/src/sql/object.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::mightbespace;
use crate::sql::common::{commas, val_char};
@ -37,6 +36,12 @@ impl From<BTreeMap<String, Value>> for Object {
}
}
impl From<HashMap<&str, Value>> for Object {
fn from(v: HashMap<&str, Value>) -> Self {
Self(v.into_iter().map(|(key, val)| (key.to_string(), val)).collect())
}
}
impl From<HashMap<String, Value>> for Object {
fn from(v: HashMap<String, Value>) -> Self {
Self(v.into_iter().collect())
@ -119,16 +124,10 @@ impl Object {
impl Object {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
let mut x = BTreeMap::new();
for (k, v) in self.iter() {
match v.compute(ctx, opt, txn, doc).await {
match v.compute(ctx, opt).await {
Ok(v) => x.insert(k.clone(), v),
Err(e) => return Err(e),
};

19
lib/src/sql/param.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::error::IResult;
use crate::sql::ident::{ident, Ident};
@ -44,32 +43,26 @@ impl Deref for Param {
impl Param {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Find the variable by name
match self.as_str() {
// This is a special param
"this" | "self" => match doc {
"this" | "self" => match ctx.doc() {
// The base document exists
Some(v) => v.compute(ctx, opt, txn, doc).await,
Some(v) => v.compute(ctx, opt).await,
// The base document does not exist
None => Ok(Value::None),
},
// This is a normal param
v => match ctx.value(v) {
// The param has been set locally
Some(v) => v.compute(ctx, opt, txn, doc).await,
Some(v) => v.compute(ctx, opt).await,
// The param has not been set locally
None => {
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Get the param definition
let val = run.get_pa(opt.ns(), opt.db(), v).await;
// Check if the param has been set globally

17
lib/src/sql/range.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::error::IResult;
use crate::sql::id::{id, Id};
@ -49,23 +48,17 @@ impl TryFrom<&str> for Range {
impl Range {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
Ok(Value::Range(Box::new(Range {
tb: self.tb.clone(),
beg: match &self.beg {
Bound::Included(id) => Bound::Included(id.compute(ctx, opt, txn, doc).await?),
Bound::Excluded(id) => Bound::Excluded(id.compute(ctx, opt, txn, doc).await?),
Bound::Included(id) => Bound::Included(id.compute(ctx, opt).await?),
Bound::Excluded(id) => Bound::Excluded(id.compute(ctx, opt).await?),
Bound::Unbounded => Bound::Unbounded,
},
end: match &self.end {
Bound::Included(id) => Bound::Included(id.compute(ctx, opt, txn, doc).await?),
Bound::Excluded(id) => Bound::Excluded(id.compute(ctx, opt, txn, doc).await?),
Bound::Included(id) => Bound::Included(id.compute(ctx, opt).await?),
Bound::Excluded(id) => Bound::Excluded(id.compute(ctx, opt).await?),
Bound::Unbounded => Bound::Unbounded,
},
})))

70
lib/src/sql/scoring.rs
View file

@ -1,29 +1,65 @@
use crate::sql::common::{closeparentheses, commas, openparentheses};
use crate::sql::error::IResult;
use crate::sql::number::number;
use crate::sql::Number;
use crate::sql::Error::Parser;
use nom::branch::alt;
use nom::bytes::complete::tag_no_case;
use nom::combinator::map;
use nom::number::complete::recognize_float;
use nom::Err::Failure;
use serde::{Deserialize, Serialize};
use std::fmt;
use std::hash::{Hash, Hasher};
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Hash)]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum Scoring {
Bm {
k1: Number,
b: Number,
order: Number,
k1: f32,
b: f32,
}, // BestMatching25
Vs, // VectorSearch
}
impl Eq for Scoring {}
impl PartialEq for Scoring {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(
Scoring::Bm {
k1,
b,
},
Scoring::Bm {
k1: other_k1,
b: other_b,
},
) => k1.to_bits() == other_k1.to_bits() && b.to_bits() == other_b.to_bits(),
(Scoring::Vs, Scoring::Vs) => true,
_ => false,
}
}
}
impl Hash for Scoring {
fn hash<H: Hasher>(&self, state: &mut H) {
match self {
Scoring::Bm {
k1,
b,
} => {
k1.to_bits().hash(state);
b.to_bits().hash(state);
}
Scoring::Vs => 0.hash(state),
}
}
}
impl Default for Scoring {
fn default() -> Self {
Self::Bm {
k1: Number::Float(1.2),
b: Number::Float(0.75),
order: Number::Int(1000),
k1: 1.2,
b: 0.75,
}
}
}
@ -34,8 +70,7 @@ impl fmt::Display for Scoring {
Self::Bm {
k1,
b,
order,
} => write!(f, "BM25({},{},{})", k1, b, order),
} => write!(f, "BM25({},{})", k1, b),
Self::Vs => f.write_str("VS"),
}
}
@ -45,18 +80,17 @@ pub fn scoring(i: &str) -> IResult<&str, Scoring> {
alt((map(tag_no_case("VS"), |_| Scoring::Vs), |i| {
let (i, _) = tag_no_case("BM25")(i)?;
let (i, _) = openparentheses(i)?;
let (i, k1) = number(i)?;
let (i, k1) = recognize_float(i)?;
let k1 = k1.parse::<f32>().map_err(|_| Failure(Parser(i)))?;
let (i, _) = commas(i)?;
let (i, b) = number(i)?;
let (i, _) = commas(i)?;
let (i, order) = number(i)?;
let (i, b) = recognize_float(i)?;
let b = b.parse::<f32>().map_err(|_| Failure(Parser(i)))?;
let (i, _) = closeparentheses(i)?;
Ok((
i,
Scoring::Bm {
k1,
b,
order,
},
))
}))(i)
@ -68,11 +102,11 @@ mod tests {
#[test]
fn scoring_bm_25() {
let sql = "BM25(1.0,0.6,100)";
let sql = "BM25(1.0,0.6)";
let res = scoring(sql);
assert!(res.is_ok());
let out = res.unwrap().1;
assert_eq!("BM25(1f,0.6f,100)", format!("{}", out))
assert_eq!("BM25(1,0.6)", format!("{}", out))
}
#[test]

11
lib/src/sql/start.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::shouldbespace;
use crate::sql::error::IResult;
@ -16,14 +15,8 @@ use std::fmt;
pub struct Start(pub Value);
impl Start {
pub(crate) async fn process(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<usize, Error> {
match self.0.compute(ctx, opt, txn, doc).await {
pub(crate) async fn process(&self, ctx: &Context<'_>, opt: &Options) -> Result<usize, Error> {
match self.0.compute(ctx, opt).await {
// This is a valid starting number
Ok(Value::Number(Number::Int(v))) if v >= 0 => Ok(v as usize),
// An invalid value was specified

45
lib/src/sql/statement.rs
View file

@ -1,12 +1,12 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::{comment, mightbespace};
use crate::sql::common::colons;
use crate::sql::error::IResult;
use crate::sql::fmt::Fmt;
use crate::sql::fmt::Pretty;
use crate::sql::statements::analyze::{analyze, AnalyzeStatement};
use crate::sql::statements::begin::{begin, BeginStatement};
use crate::sql::statements::cancel::{cancel, CancelStatement};
use crate::sql::statements::commit::{commit, CommitStatement};
@ -74,6 +74,7 @@ pub fn statements(i: &str) -> IResult<&str, Statements> {
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Store, Hash)]
pub enum Statement {
Analyze(AnalyzeStatement),
Begin(BeginStatement),
Cancel(CancelStatement),
Commit(CommitStatement),
@ -112,6 +113,7 @@ impl Statement {
/// Check if we require a writeable transaction
pub(crate) fn writeable(&self) -> bool {
match self {
Self::Analyze(_) => false,
Self::Create(v) => v.writeable(),
Self::Define(_) => true,
Self::Delete(v) => v.writeable(),
@ -133,29 +135,24 @@ impl Statement {
}
}
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
match self {
Self::Create(v) => v.compute(ctx, opt, txn, doc).await,
Self::Delete(v) => v.compute(ctx, opt, txn, doc).await,
Self::Define(v) => v.compute(ctx, opt, txn, doc).await,
Self::Ifelse(v) => v.compute(ctx, opt, txn, doc).await,
Self::Info(v) => v.compute(ctx, opt, txn, doc).await,
Self::Insert(v) => v.compute(ctx, opt, txn, doc).await,
Self::Kill(v) => v.compute(ctx, opt, txn, doc).await,
Self::Live(v) => v.compute(ctx, opt, txn, doc).await,
Self::Output(v) => v.compute(ctx, opt, txn, doc).await,
Self::Relate(v) => v.compute(ctx, opt, txn, doc).await,
Self::Remove(v) => v.compute(ctx, opt, txn, doc).await,
Self::Select(v) => v.compute(ctx, opt, txn, doc).await,
Self::Set(v) => v.compute(ctx, opt, txn, doc).await,
Self::Sleep(v) => v.compute(ctx, opt, txn, doc).await,
Self::Update(v) => v.compute(ctx, opt, txn, doc).await,
Self::Analyze(v) => v.compute(ctx, opt).await,
Self::Create(v) => v.compute(ctx, opt).await,
Self::Delete(v) => v.compute(ctx, opt).await,
Self::Define(v) => v.compute(ctx, opt).await,
Self::Ifelse(v) => v.compute(ctx, opt).await,
Self::Info(v) => v.compute(ctx, opt).await,
Self::Insert(v) => v.compute(ctx, opt).await,
Self::Kill(v) => v.compute(ctx, opt).await,
Self::Live(v) => v.compute(ctx, opt).await,
Self::Output(v) => v.compute(ctx, opt).await,
Self::Relate(v) => v.compute(ctx, opt).await,
Self::Remove(v) => v.compute(ctx, opt).await,
Self::Select(v) => v.compute(ctx, opt).await,
Self::Set(v) => v.compute(ctx, opt).await,
Self::Sleep(v) => v.compute(ctx, opt).await,
Self::Update(v) => v.compute(ctx, opt).await,
_ => unreachable!(),
}
}
@ -164,6 +161,7 @@ impl Statement {
impl Display for Statement {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
match self {
Self::Analyze(v) => write!(Pretty::from(f), "{v}"),
Self::Begin(v) => write!(Pretty::from(f), "{v}"),
Self::Cancel(v) => write!(Pretty::from(f), "{v}"),
Self::Commit(v) => write!(Pretty::from(f), "{v}"),
@ -192,6 +190,7 @@ pub fn statement(i: &str) -> IResult<&str, Statement> {
delimited(
mightbespace,
alt((
map(analyze, Statement::Analyze),
map(begin, Statement::Begin),
map(cancel, Statement::Cancel),
map(commit, Statement::Commit),

99
lib/src/sql/statements/analyze.rs Normal file
View file

@ -0,0 +1,99 @@
use crate::ctx::Context;
use crate::dbs::Level;
use crate::dbs::Options;
use crate::err::Error;
use crate::idx::ft::FtIndex;
use crate::idx::IndexKeyBase;
use crate::sql::comment::shouldbespace;
use crate::sql::error::IResult;
use crate::sql::ident::{ident, Ident};
use crate::sql::index::Index;
use crate::sql::value::Value;
use derive::Store;
use nom::bytes::complete::tag_no_case;
use serde::{Deserialize, Serialize};
use std::fmt;
use std::fmt::{Display, Formatter};
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Store, Hash)]
pub enum AnalyzeStatement {
Idx(Ident, Ident),
}
impl AnalyzeStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
match self {
AnalyzeStatement::Idx(tb, idx) => {
// Selected DB?
opt.needs(Level::Db)?;
// Allowed to run?
opt.check(Level::Db)?;
// Clone transaction
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = txn.lock().await;
// Read the index
let ix = run.get_ix(opt.ns(), opt.db(), tb.as_str(), idx.as_str()).await?;
let ikb = IndexKeyBase::new(opt, &ix);
// Index operation dispatching
let stats = match &ix.index {
Index::Search {
az,
order,
..
} => {
let az = run.get_az(opt.ns(), opt.db(), az.as_str()).await?;
let ft = FtIndex::new(&mut run, az, ikb, *order).await?;
ft.statistics(&mut run).await?
}
_ => {
return Err(Error::FeatureNotYetImplemented {
feature: "Statistics on unique and non-unique indexes.",
})
}
};
// Return the result object
Value::from(stats).ok()
}
}
}
}
pub fn analyze(i: &str) -> IResult<&str, AnalyzeStatement> {
let (i, _) = tag_no_case("ANALYZE")(i)?;
let (i, _) = shouldbespace(i)?;
let (i, _) = tag_no_case("INDEX")(i)?;
let (i, _) = shouldbespace(i)?;
let (i, idx) = ident(i)?;
let (i, _) = shouldbespace(i)?;
let (i, _) = tag_no_case("ON")(i)?;
let (i, _) = shouldbespace(i)?;
let (i, tb) = ident(i)?;
Ok((i, AnalyzeStatement::Idx(tb, idx)))
}
impl Display for AnalyzeStatement {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
match self {
Self::Idx(tb, idx) => write!(f, "ANALYZE INDEX {idx} ON {tb}"),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn analyze_index() {
let sql = "ANALYZE INDEX my_index ON my_table";
let res = analyze(sql);
assert!(res.is_ok());
let out = res.unwrap().1;
assert_eq!(out, AnalyzeStatement::Idx(Ident::from("my_table"), Ident::from("my_index")));
assert_eq!("ANALYZE INDEX my_index ON my_table", format!("{}", out));
}
}

15
lib/src/sql/statements/create.rs
View file

@ -4,7 +4,6 @@ use crate::dbs::Iterator;
use crate::dbs::Level;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::shouldbespace;
use crate::sql::data::{data, Data};
@ -43,13 +42,7 @@ impl CreateStatement {
}
}
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Selected DB?
opt.needs(Level::Db)?;
// Allowed to run?
@ -60,11 +53,11 @@ impl CreateStatement {
let opt = &opt.futures(false);
// Loop over the create targets
for w in self.what.0.iter() {
let v = w.compute(ctx, opt, txn, doc).await?;
let v = w.compute(ctx, opt).await?;
match v {
Value::Table(v) => match &self.data {
// There is a data clause so check for a record id
Some(data) => match data.rid(ctx, opt, txn, &v).await {
Some(data) => match data.rid(ctx, opt, &v).await {
// There was a problem creating the record id
Err(e) => return Err(e),
// There is an id field so use the record id
@ -123,7 +116,7 @@ impl CreateStatement {
// Assign the statement
let stm = Statement::from(self);
// Output the results
i.output(ctx, opt, txn, &stm).await
i.output(ctx, opt, &stm).await
}
}

219
lib/src/sql/statements/define.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Level;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::algorithm::{algorithm, Algorithm};
use crate::sql::base::{base, base_or_scope, Base};
@ -62,26 +61,20 @@ pub enum DefineStatement {
impl DefineStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
match self {
Self::Namespace(ref v) => v.compute(ctx, opt, txn, doc).await,
Self::Database(ref v) => v.compute(ctx, opt, txn, doc).await,
Self::Function(ref v) => v.compute(ctx, opt, txn, doc).await,
Self::Login(ref v) => v.compute(ctx, opt, txn, doc).await,
Self::Token(ref v) => v.compute(ctx, opt, txn, doc).await,
Self::Scope(ref v) => v.compute(ctx, opt, txn, doc).await,
Self::Param(ref v) => v.compute(ctx, opt, txn, doc).await,
Self::Table(ref v) => v.compute(ctx, opt, txn, doc).await,
Self::Event(ref v) => v.compute(ctx, opt, txn, doc).await,
Self::Field(ref v) => v.compute(ctx, opt, txn, doc).await,
Self::Index(ref v) => v.compute(ctx, opt, txn, doc).await,
Self::Analyzer(ref v) => v.compute(ctx, opt, txn, doc).await,
Self::Namespace(ref v) => v.compute(ctx, opt).await,
Self::Database(ref v) => v.compute(ctx, opt).await,
Self::Function(ref v) => v.compute(ctx, opt).await,
Self::Login(ref v) => v.compute(ctx, opt).await,
Self::Token(ref v) => v.compute(ctx, opt).await,
Self::Scope(ref v) => v.compute(ctx, opt).await,
Self::Param(ref v) => v.compute(ctx, opt).await,
Self::Table(ref v) => v.compute(ctx, opt).await,
Self::Event(ref v) => v.compute(ctx, opt).await,
Self::Field(ref v) => v.compute(ctx, opt).await,
Self::Index(ref v) => v.compute(ctx, opt).await,
Self::Analyzer(ref v) => v.compute(ctx, opt).await,
}
}
}
@ -134,20 +127,18 @@ pub struct DefineNamespaceStatement {
impl DefineNamespaceStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
_ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// No need for NS/DB
opt.needs(Level::Kv)?;
// Allowed to run?
opt.check(Level::Kv)?;
// Process the statement
let key = crate::key::ns::new(&self.name);
txn.clone().lock().await.set(key, self).await?;
// Clone transaction
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = txn.lock().await;
run.set(key, self).await?;
// Ok all good
Ok(Value::None)
}
@ -185,21 +176,15 @@ pub struct DefineDatabaseStatement {
impl DefineDatabaseStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
_ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Selected NS?
opt.needs(Level::Ns)?;
// Allowed to run?
opt.check(Level::Ns)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the statement
let key = crate::key::db::new(opt.ns(), &self.name);
run.add_ns(opt.ns(), opt.strict).await?;
@ -243,21 +228,15 @@ pub struct DefineFunctionStatement {
impl DefineFunctionStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
_ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Selected DB?
opt.needs(Level::Db)?;
// Allowed to run?
opt.check(Level::Db)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the statement
let key = crate::key::fc::new(opt.ns(), opt.db(), &self.name);
run.add_ns(opt.ns(), opt.strict).await?;
@ -327,29 +306,23 @@ pub struct DefineAnalyzerStatement {
}
impl DefineAnalyzerStatement {
pub(crate) async fn compute(
&self,
_ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Selected DB?
opt.needs(Level::Db)?;
// Allowed to run?
opt.check(Level::Db)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the statement
let key = crate::key::az::new(opt.ns(), opt.db(), &self.name);
run.add_ns(opt.ns(), opt.strict).await?;
run.add_db(opt.ns(), opt.db(), opt.strict).await?;
run.set(key, self).await?;
// Release the transaction
drop(run);
// Ok all good
drop(run); // Do we really need this?
// Ok all good
Ok(Value::None)
}
}
@ -369,7 +342,7 @@ impl Display for DefineAnalyzerStatement {
}
}
fn analyzer(i: &str) -> IResult<&str, DefineAnalyzerStatement> {
pub(crate) fn analyzer(i: &str) -> IResult<&str, DefineAnalyzerStatement> {
let (i, _) = tag_no_case("DEFINE")(i)?;
let (i, _) = shouldbespace(i)?;
let (i, _) = tag_no_case("ANALYZER")(i)?;
@ -404,13 +377,7 @@ pub struct DefineLoginStatement {
impl DefineLoginStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
_ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
match self.base {
Base::Ns => {
// Selected DB?
@ -418,9 +385,9 @@ impl DefineLoginStatement {
// Allowed to run?
opt.check(Level::Kv)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the statement
let key = crate::key::nl::new(opt.ns(), &self.name);
run.add_ns(opt.ns(), opt.strict).await?;
@ -434,9 +401,9 @@ impl DefineLoginStatement {
// Allowed to run?
opt.check(Level::Ns)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the statement
let key = crate::key::dl::new(opt.ns(), opt.db(), &self.name);
run.add_ns(opt.ns(), opt.strict).await?;
@ -529,13 +496,7 @@ pub struct DefineTokenStatement {
impl DefineTokenStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
_ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
match &self.base {
Base::Ns => {
// Selected DB?
@ -543,9 +504,9 @@ impl DefineTokenStatement {
// Allowed to run?
opt.check(Level::Kv)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the statement
let key = crate::key::nt::new(opt.ns(), &self.name);
run.add_ns(opt.ns(), opt.strict).await?;
@ -559,9 +520,9 @@ impl DefineTokenStatement {
// Allowed to run?
opt.check(Level::Ns)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the statement
let key = crate::key::dt::new(opt.ns(), opt.db(), &self.name);
run.add_ns(opt.ns(), opt.strict).await?;
@ -576,9 +537,9 @@ impl DefineTokenStatement {
// Allowed to run?
opt.check(Level::Db)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the statement
let key = crate::key::st::new(opt.ns(), opt.db(), sc, &self.name);
run.add_ns(opt.ns(), opt.strict).await?;
@ -651,21 +612,15 @@ pub struct DefineScopeStatement {
impl DefineScopeStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
_ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Selected DB?
opt.needs(Level::Db)?;
// Allowed to run?
opt.check(Level::Db)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the statement
let key = crate::key::sc::new(opt.ns(), opt.db(), &self.name);
run.add_ns(opt.ns(), opt.strict).await?;
@ -772,21 +727,15 @@ pub struct DefineParamStatement {
impl DefineParamStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
_ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Selected DB?
opt.needs(Level::Db)?;
// Allowed to run?
opt.check(Level::Db)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the statement
let key = crate::key::pa::new(opt.ns(), opt.db(), &self.name);
run.add_ns(opt.ns(), opt.strict).await?;
@ -838,21 +787,15 @@ pub struct DefineTableStatement {
}
impl DefineTableStatement {
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Selected DB?
opt.needs(Level::Db)?;
// Allowed to run?
opt.check(Level::Db)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the statement
let key = crate::key::tb::new(opt.ns(), opt.db(), &self.name);
run.add_ns(opt.ns(), opt.strict).await?;
@ -889,7 +832,7 @@ impl DefineTableStatement {
what: Values(vec![Value::Table(v.clone())]),
..UpdateStatement::default()
};
stm.compute(ctx, opt, txn, doc).await?;
stm.compute(ctx, opt).await?;
}
}
// Ok all good
@ -1023,21 +966,15 @@ pub struct DefineEventStatement {
impl DefineEventStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
_ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Selected DB?
opt.needs(Level::Db)?;
// Allowed to run?
opt.check(Level::Db)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the statement
let key = crate::key::ev::new(opt.ns(), opt.db(), &self.what, &self.name);
run.add_ns(opt.ns(), opt.strict).await?;
@ -1110,21 +1047,15 @@ pub struct DefineFieldStatement {
impl DefineFieldStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
_ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Selected DB?
opt.needs(Level::Db)?;
// Allowed to run?
opt.check(Level::Db)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the statement
let fd = self.name.to_string();
let key = crate::key::fd::new(opt.ns(), opt.db(), &self.what, &fd);
@ -1273,21 +1204,15 @@ pub struct DefineIndexStatement {
impl DefineIndexStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Selected DB?
opt.needs(Level::Db)?;
// Allowed to run?
opt.check(Level::Db)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the statement
let key = crate::key::ix::new(opt.ns(), opt.db(), &self.what, &self.name);
run.add_ns(opt.ns(), opt.strict).await?;
@ -1316,7 +1241,7 @@ impl DefineIndexStatement {
what: Values(vec![Value::Table(self.what.clone().into())]),
..UpdateStatement::default()
};
stm.compute(ctx, opt, txn, doc).await?;
stm.compute(ctx, opt).await?;
// Ok all good
Ok(Value::None)
}
@ -1364,7 +1289,7 @@ fn index(i: &str) -> IResult<&str, DefineIndexStatement> {
mod tests {
use super::*;
use crate::sql::scoring::Scoring;
use crate::sql::{Number, Part};
use crate::sql::Part;
#[test]
fn check_define_serialize() {
@ -1408,7 +1333,7 @@ mod tests {
#[test]
fn check_create_search_index_with_highlights() {
let sql = "DEFINE INDEX my_index ON TABLE my_table COLUMNS my_col SEARCH my_analyzer BM25(1.2,0.75,1000) HIGHLIGHTS";
let sql = "DEFINE INDEX my_index ON TABLE my_table COLUMNS my_col SEARCH ANALYZER my_analyzer BM25(1.2,0.75) ORDER 1000 HIGHLIGHTS";
let (_, idx) = index(sql).unwrap();
assert_eq!(
idx,
@ -1420,19 +1345,20 @@ mod tests {
az: Ident("my_analyzer".to_string()),
hl: true,
sc: Scoring::Bm {
k1: Number::Float(1.2),
b: Number::Float(0.75),
order: Number::Int(1000)
k1: 1.2,
b: 0.75,
},
order: 1000
},
}
);
assert_eq!(idx.to_string(), "DEFINE INDEX my_index ON my_table FIELDS my_col SEARCH my_analyzer BM25(1.2f,0.75f,1000) HIGHLIGHTS");
assert_eq!(idx.to_string(), "DEFINE INDEX my_index ON my_table FIELDS my_col SEARCH ANALYZER my_analyzer BM25(1.2,0.75) ORDER 1000 HIGHLIGHTS");
}
#[test]
fn check_create_search_index() {
let sql = "DEFINE INDEX my_index ON TABLE my_table COLUMNS my_col SEARCH my_analyzer VS";
let sql =
"DEFINE INDEX my_index ON TABLE my_table COLUMNS my_col SEARCH ANALYZER my_analyzer VS";
let (_, idx) = index(sql).unwrap();
assert_eq!(
idx,
@ -1444,12 +1370,13 @@ mod tests {
az: Ident("my_analyzer".to_string()),
hl: false,
sc: Scoring::Vs,
order: 100
},
}
);
assert_eq!(
idx.to_string(),
"DEFINE INDEX my_index ON my_table FIELDS my_col SEARCH my_analyzer VS"
"DEFINE INDEX my_index ON my_table FIELDS my_col SEARCH ANALYZER my_analyzer VS ORDER 100"
);
}
}

13
lib/src/sql/statements/delete.rs
View file

@ -4,7 +4,6 @@ use crate::dbs::Iterator;
use crate::dbs::Level;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::shouldbespace;
use crate::sql::cond::{cond, Cond};
@ -43,13 +42,7 @@ impl DeleteStatement {
}
}
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Selected DB?
opt.needs(Level::Db)?;
// Allowed to run?
@ -60,7 +53,7 @@ impl DeleteStatement {
let opt = &opt.futures(false);
// Loop over the delete targets
for w in self.what.0.iter() {
let v = w.compute(ctx, opt, txn, doc).await?;
let v = w.compute(ctx, opt).await?;
match v {
Value::Table(v) => i.ingest(Iterable::Table(v)),
Value::Thing(v) => i.ingest(Iterable::Thing(v)),
@ -116,7 +109,7 @@ impl DeleteStatement {
// Assign the statement
let stm = Statement::from(self);
// Output the results
i.output(ctx, opt, txn, &stm).await
i.output(ctx, opt, &stm).await
}
}

15
lib/src/sql/statements/ifelse.rs
View file

@ -1,6 +1,5 @@
use crate::ctx::Context;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::shouldbespace;
use crate::sql::error::IResult;
@ -30,21 +29,15 @@ impl IfelseStatement {
self.close.as_ref().map_or(false, |v| v.writeable())
}
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
for (ref cond, ref then) in &self.exprs {
let v = cond.compute(ctx, opt, txn, doc).await?;
let v = cond.compute(ctx, opt).await?;
if v.is_truthy() {
return then.compute(ctx, opt, txn, doc).await;
return then.compute(ctx, opt).await;
}
}
match self.close {
Some(ref v) => v.compute(ctx, opt, txn, doc).await,
Some(ref v) => v.compute(ctx, opt).await,
None => Ok(Value::None),
}
}

33
lib/src/sql/statements/info.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Level;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::shouldbespace;
use crate::sql::error::IResult;
@ -25,13 +24,7 @@ pub enum InfoStatement {
impl InfoStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
_ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Allowed to run?
match self {
InfoStatement::Kv => {
@ -39,12 +32,12 @@ impl InfoStatement {
opt.needs(Level::Kv)?;
// Allowed to run?
opt.check(Level::Kv)?;
// Clone transaction
let run = txn.clone();
// Claim transaction
let mut run = run.lock().await;
// Create the result set
let mut res = Object::default();
// Clone transaction
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = txn.lock().await;
// Process the statement
let mut tmp = Object::default();
for v in run.all_ns().await?.iter() {
@ -60,9 +53,9 @@ impl InfoStatement {
// Allowed to run?
opt.check(Level::Ns)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Create the result set
let mut res = Object::default();
// Process the databases
@ -92,9 +85,9 @@ impl InfoStatement {
// Allowed to run?
opt.check(Level::Db)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Create the result set
let mut res = Object::default();
// Process the logins
@ -148,9 +141,9 @@ impl InfoStatement {
// Allowed to run?
opt.check(Level::Db)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Create the result set
let mut res = Object::default();
// Process the tokens
@ -168,9 +161,9 @@ impl InfoStatement {
// Allowed to run?
opt.check(Level::Db)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Create the result set
let mut res = Object::default();
// Process the events

17
lib/src/sql/statements/insert.rs
View file

@ -4,7 +4,6 @@ use crate::dbs::Iterator;
use crate::dbs::Level;
use crate::dbs::Options;
use crate::dbs::Statement;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::shouldbespace;
use crate::sql::data::{single, update, values, Data};
@ -46,13 +45,7 @@ impl InsertStatement {
}
}
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Selected DB?
opt.needs(Level::Db)?;
// Allowed to run?
@ -70,8 +63,8 @@ impl InsertStatement {
let mut o = Value::base();
// Set each field from the expression
for (k, v) in v.iter() {
let v = v.compute(ctx, opt, txn, None).await?;
o.set(ctx, opt, txn, k, v).await?;
let v = v.compute(ctx, opt).await?;
o.set(ctx, opt, k, v).await?;
}
// Specify the new table record id
let id = o.rid().generate(&self.into, true)?;
@ -81,7 +74,7 @@ impl InsertStatement {
}
// Check if this is a modern statement
Data::SingleExpression(v) => {
let v = v.compute(ctx, opt, txn, doc).await?;
let v = v.compute(ctx, opt).await?;
match v {
Value::Array(v) => {
for v in v {
@ -109,7 +102,7 @@ impl InsertStatement {
// Assign the statement
let stm = Statement::from(self);
// Output the results
i.output(ctx, opt, txn, &stm).await
i.output(ctx, opt, &stm).await
}
}

13
lib/src/sql/statements/kill.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Level;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::shouldbespace;
use crate::sql::error::IResult;
@ -19,13 +18,7 @@ pub struct KillStatement {
impl KillStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
_ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
_doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Allowed to run?
opt.realtime()?;
// Selected DB?
@ -33,9 +26,9 @@ impl KillStatement {
// Allowed to run?
opt.check(Level::No)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Create the live query key
let key = crate::key::lq::new(opt.ns(), opt.db(), &self.id);
// Fetch the live query key if it exists

15
lib/src/sql/statements/live.rs
View file

@ -1,7 +1,6 @@
use crate::ctx::Context;
use crate::dbs::Level;
use crate::dbs::Options;
use crate::dbs::Transaction;
use crate::err::Error;
use crate::sql::comment::shouldbespace;
use crate::sql::cond::{cond, Cond};
@ -32,13 +31,7 @@ pub struct LiveStatement {
impl LiveStatement {
/// Process this type returning a computed simple Value
pub(crate) async fn compute(
&self,
ctx: &Context<'_>,
opt: &Options,
txn: &Transaction,
doc: Option<&Value>,
) -> Result<Value, Error> {
pub(crate) async fn compute(&self, ctx: &Context<'_>, opt: &Options) -> Result<Value, Error> {
// Allowed to run?
opt.realtime()?;
// Selected DB?
@ -46,11 +39,11 @@ impl LiveStatement {
// Allowed to run?
opt.check(Level::No)?;
// Clone transaction
let run = txn.clone();
let txn = ctx.clone_transaction()?;
// Claim transaction
let mut run = run.lock().await;
let mut run = txn.lock().await;
// Process the live query table
match self.what.compute(ctx, opt, txn, doc).await? {
match self.what.compute(ctx, opt).await? {
Value::Table(tb) => {
// Insert the live query
let key = crate::key::lq::new(opt.ns(), opt.db(), &self.id);

1
lib/src/sql/statements/mod.rs
View file

@ -1,3 +1,4 @@
pub(crate) mod analyze;
pub(crate) mod begin;
pub(crate) mod cancel;
pub(crate) mod commit;

Some files were not shown because too many files have changed in this diff Show more