Search failed when analyzer does not have tokenizers (#4798)
This commit is contained in:
parent
9c155fd317
commit
7d7f3a20fa
2 changed files with 29 additions and 7 deletions
|
@ -278,13 +278,16 @@ impl Analyzer {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(t) = &self.az.tokenizers {
|
if input.is_empty() {
|
||||||
if !input.is_empty() {
|
return Ok(Tokens::new(input));
|
||||||
let t = Tokenizer::tokenize(t, input);
|
|
||||||
return Filter::apply_filters(t, &self.filters, stage);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Ok(Tokens::new(input))
|
|
||||||
|
let tokens = if let Some(t) = &self.az.tokenizers {
|
||||||
|
Tokenizer::tokenize(t, input)
|
||||||
|
} else {
|
||||||
|
Tokenizer::tokenize(&[], input)
|
||||||
|
};
|
||||||
|
Filter::apply_filters(tokens, &self.filters, stage)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Used for exposing the analyzer as the native function `search::analyze`
|
/// Used for exposing the analyzer as the native function `search::analyze`
|
||||||
|
@ -351,4 +354,9 @@ mod tests {
|
||||||
let tokens = get_analyzer_tokens(def, input).await;
|
let tokens = get_analyzer_tokens(def, input).await;
|
||||||
assert_eq!(tokens.list(), expected);
|
assert_eq!(tokens.list(), expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_no_tokenizer() {
|
||||||
|
test_analyzer("ANALYZER test FILTERS lowercase", "ab", &["ab"]).await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
mod parse;
|
mod parse;
|
||||||
use parse::Parse;
|
use parse::Parse;
|
||||||
mod helpers;
|
mod helpers;
|
||||||
use crate::helpers::skip_ok;
|
use crate::helpers::{skip_ok, Test};
|
||||||
use helpers::new_ds;
|
use helpers::new_ds;
|
||||||
use surrealdb::dbs::Session;
|
use surrealdb::dbs::Session;
|
||||||
use surrealdb::err::Error;
|
use surrealdb::err::Error;
|
||||||
|
@ -704,3 +704,17 @@ async fn select_where_matches_mixing_indexes() -> Result<(), Error> {
|
||||||
assert_eq!(format!("{:#}", tmp), format!("{:#}", val));
|
assert_eq!(format!("{:#}", tmp), format!("{:#}", val));
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn select_where_matches_analyser_without_tokenizer() -> Result<(), Error> {
|
||||||
|
let sql = r"
|
||||||
|
DEFINE ANALYZER az FILTERS lowercase,ngram(1,5);
|
||||||
|
CREATE t:1 SET text = 'ab';
|
||||||
|
DEFINE INDEX search_idx ON TABLE t COLUMNS text SEARCH ANALYZER az BM25 HIGHLIGHTS;
|
||||||
|
SELECT * FROM t WHERE text @@ 'a';";
|
||||||
|
let mut t = Test::new(sql).await?;
|
||||||
|
t.expect_size(4)?;
|
||||||
|
t.skip_ok(3)?;
|
||||||
|
t.expect_val("[{ id: t:1, text: 'ab' }]")?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in a new issue