noth/surrealpatch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Search failed when analyzer does not have tokenizers (#4798)

Browse source
This commit is contained in:
Emmanuel Keller 2024-09-17 14:54:00 +01:00 committed by GitHub
parent 9c155fd317
commit 7d7f3a20fa
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 29 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
core/src/idx/ft/analyzer/mod.rs
View file

@ -278,13 +278,16 @@ impl Analyzer {
});
}
}
if let Some(t) = &self.az.tokenizers {
if !input.is_empty() {
let t = Tokenizer::tokenize(t, input);
return Filter::apply_filters(t, &self.filters, stage);
if input.is_empty() {
return Ok(Tokens::new(input));
}
}
Ok(Tokens::new(input))
let tokens = if let Some(t) = &self.az.tokenizers {
Tokenizer::tokenize(t, input)
} else {
Tokenizer::tokenize(&[], input)
};
Filter::apply_filters(tokens, &self.filters, stage)
}
/// Used for exposing the analyzer as the native function `search::analyze`
@ -351,4 +354,9 @@ mod tests {
let tokens = get_analyzer_tokens(def, input).await;
assert_eq!(tokens.list(), expected);
}
#[tokio::test]
async fn test_no_tokenizer() {
test_analyzer("ANALYZER test FILTERS lowercase", "ab", &["ab"]).await;
}
}

16
sdk/tests/matches.rs
View file

@ -1,7 +1,7 @@
mod parse;
use parse::Parse;
mod helpers;
use crate::helpers::skip_ok;
use crate::helpers::{skip_ok, Test};
use helpers::new_ds;
use surrealdb::dbs::Session;
use surrealdb::err::Error;
@ -704,3 +704,17 @@ async fn select_where_matches_mixing_indexes() -> Result<(), Error> {
assert_eq!(format!("{:#}", tmp), format!("{:#}", val));
Ok(())
}
#[tokio::test]
async fn select_where_matches_analyser_without_tokenizer() -> Result<(), Error> {
let sql = r"
DEFINE ANALYZER az FILTERS lowercase,ngram(1,5);
CREATE t:1 SET text = 'ab';
DEFINE INDEX search_idx ON TABLE t COLUMNS text SEARCH ANALYZER az BM25 HIGHLIGHTS;
SELECT * FROM t WHERE text @@ 'a';";
let mut t = Test::new(sql).await?;
t.expect_size(4)?;
t.skip_ok(3)?;
t.expect_val("[{ id: t:1, text: 'ab' }]")?;
Ok(())
}