Feature for #2065: Add array functions to reduce JS slowdown. (#2156)

Co-authored-by: Mees Delzenne <DelSkayn@users.noreply.github.com>
This commit is contained in:
JustAnotherCodemonkey 2023-07-10 00:15:18 -08:00 committed by GitHub
parent cf4c8c5f5d
commit a10b9cbb75
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 535 additions and 0 deletions

View file

@ -151,17 +151,28 @@
"array::all("
"array::any("
"array::append("
"array::boolean_and("
"array::boolean_not("
"array::boolean_or("
"array::boolean_xor("
"array::clump("
"array::combine("
"array::complement("
"array::concat("
"array::difference("
"array::distinct("
"array::filter_index("
"array::find_index("
"array::flatten("
"array::group("
"array::insert("
"array::intersect("
"array::join("
"array::len("
"array::logical_and("
"array::logical_or("
"array::logical_xor("
"array::matches("
"array::max("
"array::min("
"array::pop("
@ -173,6 +184,7 @@
"array::sort("
"array::sort::asc("
"array::sort::desc("
"array::transpose("
"array::union("
"count("
"crypto"

View file

@ -151,17 +151,28 @@
"array::all("
"array::any("
"array::append("
"array::boolean_and("
"array::boolean_not("
"array::boolean_or("
"array::boolean_xor("
"array::clump("
"array::combine("
"array::complement("
"array::concat("
"array::difference("
"array::distinct("
"array::filter_index("
"array::find_index("
"array::flatten("
"array::group("
"array::insert("
"array::intersect("
"array::join("
"array::len("
"array::logical_and("
"array::logical_or("
"array::logical_xor("
"array::matches("
"array::max("
"array::min("
"array::pop("
@ -173,6 +184,7 @@
"array::sort("
"array::sort::asc("
"array::sort::desc("
"array::transpose("
"array::union("
"count("
"crypto"

View file

@ -1,11 +1,14 @@
use crate::err::Error;
use crate::sql::array::Array;
use crate::sql::array::Clump;
use crate::sql::array::Combine;
use crate::sql::array::Complement;
use crate::sql::array::Concat;
use crate::sql::array::Difference;
use crate::sql::array::Flatten;
use crate::sql::array::Intersect;
use crate::sql::array::Matches;
use crate::sql::array::Transpose;
use crate::sql::array::Union;
use crate::sql::array::Uniq;
use crate::sql::value::Value;
@ -42,6 +45,54 @@ pub fn append((mut array, value): (Array, Value)) -> Result<Value, Error> {
Ok(array.into())
}
pub fn boolean_and((lh, rh): (Array, Array)) -> Result<Value, Error> {
let longest_length = lh.len().max(rh.len());
let mut results = Array::with_capacity(longest_length);
for i in 0..longest_length {
let lhv = lh.get(i);
let rhv = rh.get(i);
results.push(
(lhv.map_or(false, |v| v.is_truthy()) && rhv.map_or(false, |v| v.is_truthy())).into(),
);
}
Ok(results.into())
}
pub fn boolean_not((mut array,): (Array,)) -> Result<Value, Error> {
array.iter_mut().for_each(|v| *v = (!v.is_truthy()).into());
Ok(array.into())
}
pub fn boolean_or((lh, rh): (Array, Array)) -> Result<Value, Error> {
let longest_length = lh.len().max(rh.len());
let mut results = Array::with_capacity(longest_length);
for i in 0..longest_length {
let lhv = lh.get(i);
let rhv = rh.get(i);
results.push(
(lhv.map_or(false, |v| v.is_truthy()) || rhv.map_or(false, |v| v.is_truthy())).into(),
);
}
Ok(results.into())
}
pub fn boolean_xor((lh, rh): (Array, Array)) -> Result<Value, Error> {
let longest_length = lh.len().max(rh.len());
let mut results = Array::with_capacity(longest_length);
for i in 0..longest_length {
let lhv = lh.get(i);
let rhv = rh.get(i);
results.push(
(lhv.map_or(false, |v| v.is_truthy()) ^ rhv.map_or(false, |v| v.is_truthy())).into(),
);
}
Ok(results.into())
}
pub fn clump((array, clump_size): (Array, i64)) -> Result<Value, Error> {
Ok(array.clump(clump_size as usize).into())
}
pub fn combine((array, other): (Array, Array)) -> Result<Value, Error> {
Ok(array.combine(other).into())
}
@ -62,6 +113,29 @@ pub fn distinct((array,): (Array,)) -> Result<Value, Error> {
Ok(array.uniq().into())
}
pub fn filter_index((array, value): (Array, Value)) -> Result<Value, Error> {
Ok(array
.iter()
.enumerate()
.filter_map(|(i, v)| {
if *v == value {
Some(Value::from(i))
} else {
None
}
})
.collect::<Vec<_>>()
.into())
}
pub fn find_index((array, value): (Array, Value)) -> Result<Value, Error> {
Ok(array
.iter()
.enumerate()
.find(|(_i, v)| **v == value)
.map_or(Value::Null, |(i, _v)| i.into()))
}
pub fn flatten((array,): (Array,)) -> Result<Value, Error> {
Ok(array.flatten().into())
}
@ -105,6 +179,82 @@ pub fn len((array,): (Array,)) -> Result<Value, Error> {
Ok(array.len().into())
}
pub fn logical_and((lh, rh): (Array, Array)) -> Result<Value, Error> {
let mut result_arr = Array::with_capacity(lh.len().max(rh.len()));
let mut iters = (lh.into_iter(), rh.into_iter());
for (lhv, rhv) in std::iter::from_fn(|| {
let r = (iters.0.next(), iters.1.next());
if r.0.is_none() && r.1.is_none() {
None
} else {
Some((r.0.unwrap_or(Value::Null), r.1.unwrap_or(Value::Null)))
}
}) {
let truth = lhv.is_truthy() && rhv.is_truthy();
let r = if lhv.is_truthy() == truth {
lhv
} else if rhv.is_truthy() == truth {
rhv
} else {
truth.into()
};
result_arr.push(r);
}
Ok(result_arr.into())
}
pub fn logical_or((lh, rh): (Array, Array)) -> Result<Value, Error> {
let mut result_arr = Array::with_capacity(lh.len().max(rh.len()));
let mut iters = (lh.into_iter(), rh.into_iter());
for (lhv, rhv) in std::iter::from_fn(|| {
let r = (iters.0.next(), iters.1.next());
if r.0.is_none() && r.1.is_none() {
None
} else {
Some((r.0.unwrap_or(Value::Null), r.1.unwrap_or(Value::Null)))
}
}) {
let truth = lhv.is_truthy() || rhv.is_truthy();
let r = if lhv.is_truthy() == truth {
lhv
} else if rhv.is_truthy() == truth {
rhv
} else {
truth.into()
};
result_arr.push(r);
}
Ok(result_arr.into())
}
pub fn logical_xor((lh, rh): (Array, Array)) -> Result<Value, Error> {
let mut result_arr = Array::with_capacity(lh.len().max(rh.len()));
let mut iters = (lh.into_iter(), rh.into_iter());
for (lhv, rhv) in std::iter::from_fn(|| {
let r = (iters.0.next(), iters.1.next());
if r.0.is_none() && r.1.is_none() {
None
} else {
Some((r.0.unwrap_or(Value::Null), r.1.unwrap_or(Value::Null)))
}
}) {
let truth = lhv.is_truthy() ^ rhv.is_truthy();
let r = if lhv.is_truthy() == truth {
lhv
} else if rhv.is_truthy() == truth {
rhv
} else {
truth.into()
};
result_arr.push(r);
}
Ok(result_arr.into())
}
pub fn matches((array, compare_val): (Array, Value)) -> Result<Value, Error> {
Ok(array.matches(compare_val).into())
}
pub fn max((array,): (Array,)) -> Result<Value, Error> {
Ok(array.into_iter().max().unwrap_or_default())
}
@ -198,6 +348,10 @@ pub fn sort((mut array, order): (Array, Option<Value>)) -> Result<Value, Error>
}
}
pub fn transpose((array,): (Array,)) -> Result<Value, Error> {
Ok(array.transpose().into())
}
pub fn union((array, other): (Array, Array)) -> Result<Value, Error> {
Ok(array.union(other).into())
}

View file

@ -81,17 +81,28 @@ pub fn synchronous(ctx: &Context<'_>, name: &str, args: Vec<Value>) -> Result<Va
"array::all" => array::all,
"array::any" => array::any,
"array::append" => array::append,
"array::boolean_and" => array::boolean_and,
"array::boolean_not" => array::boolean_not,
"array::boolean_or" => array::boolean_or,
"array::boolean_xor" => array::boolean_xor,
"array::clump" => array::clump,
"array::combine" => array::combine,
"array::complement" => array::complement,
"array::concat" => array::concat,
"array::difference" => array::difference,
"array::distinct" => array::distinct,
"array::filter_index" => array::filter_index,
"array::find_index" => array::find_index,
"array::flatten" => array::flatten,
"array::group" => array::group,
"array::insert" => array::insert,
"array::intersect" => array::intersect,
"array::join" => array::join,
"array::len" => array::len,
"array::logical_and" => array::logical_and,
"array::logical_or" => array::logical_or,
"array::logical_xor" => array::logical_xor,
"array::matches" => array::matches,
"array::max" => array::max,
"array::min" => array::min,
"array::pop" => array::pop,
@ -101,6 +112,7 @@ pub fn synchronous(ctx: &Context<'_>, name: &str, args: Vec<Value>) -> Result<Va
"array::reverse" => array::reverse,
"array::slice" => array::slice,
"array::sort" => array::sort,
"array::transpose" => array::transpose,
"array::union" => array::union,
"array::sort::asc" => array::sort::asc,
"array::sort::desc" => array::sort::desc,

View file

@ -11,17 +11,28 @@ impl_module_def!(
"all" => run,
"any" => run,
"append" => run,
"boolean_and" => run,
"boolean_not" => run,
"boolean_or" => run,
"boolean_xor" => run,
"clump" => run,
"combine" => run,
"complement" => run,
"concat" => run,
"difference" => run,
"distinct" => run,
"filter_index" => run,
"find_index" => run,
"flatten" => run,
"group" => run,
"insert" => run,
"intersect" => run,
"join" => run,
"len" => run,
"logical_and" => run,
"logical_or" => run,
"logical_xor" => run,
"matches" => run,
"max" => run,
"min" => run,
"pop" => run,
@ -31,5 +42,6 @@ impl_module_def!(
"reverse" => run,
"slice" => run,
"sort" => (sort::Package),
"transpose" => run,
"union" => run
);

View file

@ -217,6 +217,22 @@ impl<T> Abolish<T> for Vec<T> {
// ------------------------------
pub(crate) trait Clump<T> {
fn clump(self, clump_size: usize) -> T;
}
impl Clump<Array> for Array {
fn clump(self, clump_size: usize) -> Array {
self.0
.chunks(clump_size)
.map::<Value, _>(|chunk| chunk.to_vec().into())
.collect::<Vec<_>>()
.into()
}
}
// ------------------------------
pub(crate) trait Combine<T> {
fn combine(self, other: T) -> T;
}
@ -325,6 +341,93 @@ impl Intersect<Self> for Array {
// ------------------------------
// Documented with the assumption that it is just for arrays.
pub(crate) trait Matches<T> {
/// Returns an array complimenting the origional where each value is true or false
/// depending on whether it is == to the compared value.
///
/// Admittedly, this is most often going to be used in `count(array::matches($arr, $val))`
/// to count the number of times an element appears in an array but it's nice to have
/// this in addition.
fn matches(self, compare_val: Value) -> T;
}
impl Matches<Array> for Array {
fn matches(self, compare_val: Value) -> Array {
self.iter().map(|arr_val| (arr_val == &compare_val).into()).collect::<Vec<Value>>().into()
}
}
// ------------------------------
// Documented with the assumption that it is just for arrays.
pub(crate) trait Transpose<T> {
/// Stacks arrays on top of each other. This can serve as 2d array transposition.
///
/// The input array can contain regular values which are treated as arrays with
/// a single element.
///
/// It's best to think of the function as creating a layered structure of the arrays
/// rather than transposing them when the input is not a 2d array. See the examples
/// for what happense when the input arrays are not all the same size.
///
/// Here's a diagram:
/// [0, 1, 2, 3], [4, 5, 6]
/// ->
/// [0 | 1 | 2 | 3]
/// [4 | 5 | 6 ]
/// ^ ^ ^ ^
/// [0, 4] [1, 5] [2, 6] [3]
///
/// # Examples
///
/// ```ignore
/// fn array(sql: &str) -> Array {
/// unimplemented!();
/// }
///
/// // Example of `transpose` doing what it says on the tin.
/// assert_eq!(array("[[0, 1], [2, 3]]").transpose(), array("[[0, 2], [1, 3]]"));
/// // `transpose` can be thought of layering arrays on top of each other so when
/// // one array runs out, it stops appearing in the output.
/// assert_eq!(array("[[0, 1], [2]]").transpose(), array("[[0, 2], [1]]"));
/// assert_eq!(array("[0, 1, 2]").transpose(), array("[[0, 1, 2]]"));
/// ```
fn transpose(self) -> T;
}
impl Transpose<Array> for Array {
fn transpose(self) -> Array {
if self.len() == 0 {
return self;
}
// I'm sure there's a way more efficient way to do this that I don't know about.
// The new array will be at *least* this large so we can start there;
let mut transposed_vec = Vec::<Value>::with_capacity(self.len());
let mut iters = self
.iter()
.map(|v| {
if let Value::Array(arr) = v {
Box::new(arr.iter().cloned()) as Box<dyn ExactSizeIterator<Item = Value>>
} else {
Box::new(std::iter::once(v).cloned())
as Box<dyn ExactSizeIterator<Item = Value>>
}
})
.collect::<Vec<_>>();
// We know there is at least one element in the array therefore iters is not empty.
// This is safe.
let longest_length = iters.iter().map(|i| i.len()).max().unwrap();
for _ in 0..longest_length {
transposed_vec
.push(iters.iter_mut().filter_map(|i| i.next()).collect::<Vec<_>>().into());
}
transposed_vec.into()
}
}
// ------------------------------
pub(crate) trait Union<T> {
fn union(self, other: T) -> T;
}
@ -414,6 +517,38 @@ mod tests {
assert_eq!(out.0.len(), 3);
}
#[test]
fn array_fnc_clump() {
fn test(input_sql: &str, clump_size: usize, expected_result: &str) {
let arr_result = array(input_sql);
assert!(arr_result.is_ok());
let arr = arr_result.unwrap().1;
let clumped_arr = arr.clump(clump_size);
assert_eq!(format!("{}", clumped_arr), expected_result);
}
test("[0, 1, 2, 3]", 2, "[[0, 1], [2, 3]]");
test("[0, 1, 2, 3, 4, 5]", 3, "[[0, 1, 2], [3, 4, 5]]");
test("[0, 1, 2]", 2, "[[0, 1], [2]]");
test("[]", 2, "[]");
}
#[test]
fn array_fnc_transpose() {
fn test(input_sql: &str, expected_result: &str) {
let arr_result = array(input_sql);
assert!(arr_result.is_ok());
let arr = arr_result.unwrap().1;
let transposed_arr = arr.transpose();
assert_eq!(format!("{}", transposed_arr), expected_result);
}
test("[[0, 1], [2, 3]]", "[[0, 2], [1, 3]]");
test("[[0, 1], [2]]", "[[0, 2], [1]]");
test("[[0, 1, 2], [true, false]]", "[[0, true], [1, false], [2]]");
test("[[0, 1], [2, 3], [4, 5]]", "[[0, 2, 4], [1, 3, 5]]");
}
#[test]
fn array_fnc_uniq_normal() {
let sql = "[1,2,1,3,3,4]";

View file

@ -279,11 +279,18 @@ fn function_array(i: &str) -> IResult<&str, &str> {
tag("all"),
tag("any"),
tag("append"),
tag("boolean_and"),
tag("boolean_not"),
tag("boolean_or"),
tag("boolean_xor"),
tag("clump"),
tag("combine"),
tag("complement"),
tag("concat"),
tag("difference"),
tag("distinct"),
tag("filter_index"),
tag("find_index"),
tag("flatten"),
tag("group"),
tag("insert"),
@ -292,17 +299,24 @@ fn function_array(i: &str) -> IResult<&str, &str> {
tag("intersect"),
tag("join"),
tag("len"),
tag("logical_and"),
tag("logical_or"),
tag("logical_xor"),
tag("matches"),
tag("max"),
tag("min"),
tag("pop"),
tag("prepend"),
tag("push"),
)),
alt((
tag("remove"),
tag("reverse"),
tag("slice"),
tag("sort::asc"),
tag("sort::desc"),
tag("sort"),
tag("transpose"),
tag("union"),
)),
))(i)

View file

@ -5,6 +5,31 @@ use surrealdb::err::Error;
use surrealdb::kvs::Datastore;
use surrealdb::sql::{Number, Value};
async fn test_queries(sql: &str, desired_responses: &[&str]) -> Result<(), Error> {
let db = Datastore::new("memory").await?;
let session = Session::for_kv().with_ns("test").with_db("test");
let response = db.execute(sql, &session, None).await?;
for (i, r) in response.into_iter().map(|r| r.result).enumerate() {
let v = r?;
if let Some(desired_response) = desired_responses.get(i) {
let desired_value = Value::parse(*desired_response);
assert_eq!(
v,
desired_value,
"Recieved responce did not match \
expected.
Query responce #{},
Desired responce: {desired_value},
Actual response: {v}",
i + 1
);
} else {
panic!("Response index {i} out of bounds of desired responses.");
}
}
Ok(())
}
// --------------------------------------------------
// array
// --------------------------------------------------
@ -147,6 +172,62 @@ async fn function_array_append() -> Result<(), Error> {
Ok(())
}
#[tokio::test]
async fn function_array_boolean_and() -> Result<(), Error> {
test_queries(
r#"RETURN array::boolean_and([false, true, false, true], [false, false, true, true]);
RETURN array::boolean_and([0, 1, 0, 1], [0, 0, 1, 1]);
RETURN array::boolean_and([true, false], [false]);
RETURN array::boolean_and([true, true], [false]);"#,
&[
"[false, false, false, true]",
"[false, false, false, true]",
"[false, false]",
"[false, false]",
],
)
.await?;
Ok(())
}
#[tokio::test]
async fn function_array_boolean_not() -> Result<(), Error> {
test_queries(
r#"RETURN array::boolean_not([false, true, 0, 1]);"#,
&["[true, false, true, false]"],
)
.await?;
Ok(())
}
#[tokio::test]
async fn function_array_boolean_or() -> Result<(), Error> {
test_queries(
r#"RETURN array::boolean_or([false, true, false, true], [false, false, true, true]);
RETURN array::boolean_or([0, 1, 0, 1], [0, 0, 1, 1]);
RETURN array::boolean_or([true, false], [false]);
RETURN array::boolean_or([true, true], [false]);"#,
&[
"[false, true, true, true]",
"[false, true, true, true]",
"[true, false]",
"[true, true]",
],
)
.await?;
Ok(())
}
#[tokio::test]
async fn function_array_boolean_xor() -> Result<(), Error> {
test_queries(
r#"RETURN array::boolean_xor([false, true, false, true], [false, false, true, true]);"#,
&["[false, true, true, false]"],
)
.await?;
Ok(())
}
#[tokio::test]
async fn function_array_combine() -> Result<(), Error> {
let sql = r#"
@ -179,6 +260,20 @@ async fn function_array_combine() -> Result<(), Error> {
Ok(())
}
#[tokio::test]
async fn function_array_clump() -> Result<(), Error> {
let sql = r#"
RETURN array::clump([0, 1, 2, 3], 2);
RETURN array::clump([0, 1, 2], 2);
RETURN array::clump([0, 1, 2], 3);
RETURN array::clump([0, 1, 2, 3, 4, 5], 3);
"#;
let desired_responses =
["[[0, 1], [2, 3]]", "[[0, 1], [2]]", "[[0, 1, 2]]", "[[0, 1, 2], [3, 4, 5]]"];
test_queries(sql, &desired_responses).await?;
Ok(())
}
#[tokio::test]
async fn function_array_complement() -> Result<(), Error> {
let sql = r#"
@ -306,6 +401,27 @@ async fn function_array_distinct() -> Result<(), Error> {
Ok(())
}
#[tokio::test]
async fn function_array_filter_index() -> Result<(), Error> {
let sql = r#"RETURN array::filter_index([0, 1, 2], 1);
RETURN array::filter_index([0, 0, 2], 0);
RETURN array::filter_index(["hello_world", "hello world", "hello wombat", "hello world"], "hello world");
RETURN array::filter_index(["nothing here"], 0);"#;
let desired_responses = ["[1]", "[0, 1]", "[1, 3]", "[]"];
test_queries(sql, &desired_responses).await?;
Ok(())
}
#[tokio::test]
async fn function_array_find_index() -> Result<(), Error> {
let sql = r#"RETURN array::find_index([5, 6, 7], 7);
RETURN array::find_index(["hello world", null, true], null);
RETURN array::find_index([0, 1, 2], 3);"#;
let desired_responses = ["2", "1", "null"];
test_queries(sql, &desired_responses).await?;
Ok(())
}
#[tokio::test]
async fn function_array_flatten() -> Result<(), Error> {
let sql = r#"
@ -503,6 +619,54 @@ async fn function_array_len() -> Result<(), Error> {
Ok(())
}
#[tokio::test]
async fn function_array_logical_and() -> Result<(), Error> {
test_queries(
r#"RETURN array::logical_and([true, false, true, false], [true, true, false, false]);
RETURN array::logical_and([1, 0, 1, 0], ["true", "true", "false", "false"]);
RETURN array::logical_and([0, 1], []);"#,
&["[true, false, false, false]", r#"[1, 0, "false", 0]"#, "[0, null]"],
)
.await?;
Ok(())
}
#[tokio::test]
async fn function_array_logical_or() -> Result<(), Error> {
test_queries(
r#"RETURN array::logical_or([true, false, true, false], [true, true, false, false]);
RETURN array::logical_or([1, 0, 1, 0], ["true", "true", "false", "false"]);
RETURN array::logical_or([0, 1], []);"#,
&["[true, true, true, false]", r#"[1, "true", 1, 0]"#, "[0, 1]"],
)
.await?;
Ok(())
}
#[tokio::test]
async fn function_array_logical_xor() -> Result<(), Error> {
test_queries(
r#"RETURN array::logical_xor([true, false, true, false], [true, true, false, false]);
RETURN array::logical_xor([1, 0, 1, 0], ["true", "true", "false", "false"]);
RETURN array::logical_xor([0, 1], []);"#,
&["[false, true, true, false]", r#"[false, "true", 1, 0]"#, "[0, 1]"],
)
.await?;
Ok(())
}
#[tokio::test]
async fn function_array_matches() -> Result<(), Error> {
test_queries(
r#"RETURN array::matches([0, 1, 2], 1);
RETURN array::matches([[], [0]], []);
RETURN array::matches([{id: "ohno:0"}, {id: "ohno:1"}], {id: "ohno:1"});"#,
&["[false, true, false]", "[true, false]", "[false, true]"],
)
.await?;
Ok(())
}
#[tokio::test]
async fn function_array_max() -> Result<(), Error> {
let sql = r#"
@ -895,6 +1059,26 @@ async fn function_array_sort_desc() -> Result<(), Error> {
Ok(())
}
#[tokio::test]
async fn function_array_transpose() -> Result<(), Error> {
let sql = r#"
RETURN array::transpose([[0, 1], [2, 3]]);
RETURN array::transpose([[0, 1, 2], [3, 4]]);
RETURN array::transpose([[0, 1], [2, 3, 4]]);
RETURN array::transpose([[0, 1], [2, 3], [4, 5]]);
RETURN array::transpose([[0, 1, 2], "oops", [null, "sorry"]]);
"#;
let desired_responses = [
"[[0, 2], [1, 3]]",
"[[0, 3], [1, 4], [2]]",
"[[0, 2], [1, 3], [4]]",
"[[0, 2, 4], [1, 3, 5]]",
"[[0, \"oops\", null], [1, \"sorry\"], [2]]",
];
test_queries(sql, &desired_responses).await?;
Ok(())
}
#[tokio::test]
async fn function_array_union() -> Result<(), Error> {
let sql = r#"