noth/surrealpatch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Feature: some more error message improvements for the new parser. (#3416)

Browse source
This commit is contained in:
Mees Delzenne 2024-02-19 16:07:39 +01:00 committed by GitHub
parent 9404688d87
commit 1e5bd504b2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 856 additions and 494 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
core/src/sql/v1/value/value.rs
View file

@ -2861,7 +2861,7 @@ mod tests {
assert_eq!(12, std::mem::size_of::<crate::sql::datetime::Datetime>()); assert_eq!(12, std::mem::size_of::<crate::sql::datetime::Datetime>());
assert_eq!(24, std::mem::size_of::<crate::sql::array::Array>()); assert_eq!(24, std::mem::size_of::<crate::sql::array::Array>());
assert_eq!(24, std::mem::size_of::<crate::sql::object::Object>()); assert_eq!(24, std::mem::size_of::<crate::sql::object::Object>());
assert!(56 >= std::mem::size_of::<crate::sql::geometry::Geometry>()); assert_eq!(56, std::mem::size_of::<crate::sql::geometry::Geometry>());
assert_eq!(24, std::mem::size_of::<crate::sql::param::Param>()); assert_eq!(24, std::mem::size_of::<crate::sql::param::Param>());
assert_eq!(24, std::mem::size_of::<crate::sql::idiom::Idiom>()); assert_eq!(24, std::mem::size_of::<crate::sql::idiom::Idiom>());
assert_eq!(24, std::mem::size_of::<crate::sql::table::Table>()); assert_eq!(24, std::mem::size_of::<crate::sql::table::Table>());

7
core/src/syn/common.rs
View file

@ -99,13 +99,13 @@ impl Location {
// Bytes of input prior to line being iteratated. // Bytes of input prior to line being iteratated.
let mut bytes_prior = 0; let mut bytes_prior = 0;
let mut iterator = LineIterator::new(source).enumerate(); let mut iterator = LineIterator::new(source).enumerate().peekable();
let start = loop { let start = loop {
let Some((line_idx, (line, seperator_offset))) = iterator.next() else { let Some((line_idx, (line, seperator_offset))) = iterator.peek() else {
panic!("tried to find location of span not belonging to string"); panic!("tried to find location of span not belonging to string");
}; };
let bytes_so_far = bytes_prior + line.len() + seperator_offset.unwrap_or(0) as usize; let bytes_so_far = bytes_prior + line.len() + seperator_offset.unwrap_or(0) as usize;
if bytes_so_far >= offset { if bytes_so_far > offset {
// found line. // found line.
let line_offset = offset - bytes_prior; let line_offset = offset - bytes_prior;
let column = if line_offset > line.len() { let column = if line_offset > line.len() {
@ -133,6 +133,7 @@ impl Location {
} }
} }
bytes_prior = bytes_so_far; bytes_prior = bytes_so_far;
iterator.next();
}; };
loop { loop {

3
core/src/syn/v2/lexer/byte.rs
View file

@ -52,10 +52,11 @@ impl<'a> Lexer<'a> {
return Err(Error::UnexpectedEof); return Err(Error::UnexpectedEof);
}; };
if let b'*' = byte { if let b'*' = byte {
let Some(byte) = self.reader.next() else { let Some(byte) = self.reader.peek() else {
return Err(Error::UnexpectedEof); return Err(Error::UnexpectedEof);
}; };
if b'/' == byte { if b'/' == byte {
self.reader.next();
self.set_whitespace_span(self.current_span()); self.set_whitespace_span(self.current_span());
self.skip_offset(); self.skip_offset();
return Ok(()); return Ok(());

4
core/src/syn/v2/lexer/ident.rs
View file

@ -55,10 +55,12 @@ impl<'a> Lexer<'a> {
// When finished parsing the identifier, try to match it to an keyword. // When finished parsing the identifier, try to match it to an keyword.
// If there is one, return it as the keyword. Original identifier can be reconstructed // If there is one, return it as the keyword. Original identifier can be reconstructed
// from the token. // from the token.
if let Some(x) = KEYWORDS.get(&UniCase::ascii(&self.scratch)).copied().flatten() { if let Some(x) = KEYWORDS.get(&UniCase::ascii(&self.scratch)).copied() {
if x != TokenKind::Identifier {
self.scratch.clear(); self.scratch.clear();
return self.finish_token(x); return self.finish_token(x);
} }
}
if self.scratch == "NaN" { if self.scratch == "NaN" {
self.scratch.clear(); self.scratch.clear();

544
core/src/syn/v2/lexer/keywords.rs
View file

@ -6,290 +6,290 @@ use phf::phf_map;
use unicase::UniCase; use unicase::UniCase;
/// A map for mapping keyword strings to a tokenkind, /// A map for mapping keyword strings to a tokenkind,
pub(crate) static KEYWORDS: phf::Map<UniCase<&'static str>, Option<TokenKind>> = phf_map! { pub(crate) static KEYWORDS: phf::Map<UniCase<&'static str>, TokenKind> = phf_map! {
// Keywords // Keywords
UniCase::ascii("AFTER") => Some(TokenKind::Keyword(Keyword::After)), UniCase::ascii("AFTER") => TokenKind::Keyword(Keyword::After),
UniCase::ascii("ALL") => Some(TokenKind::Keyword(Keyword::All)), UniCase::ascii("ALL") => TokenKind::Keyword(Keyword::All),
UniCase::ascii("ANALYZE") => Some(TokenKind::Keyword(Keyword::Analyze)), UniCase::ascii("ANALYZE") => TokenKind::Keyword(Keyword::Analyze),
UniCase::ascii("ANALYZER") => Some(TokenKind::Keyword(Keyword::Analyzer)), UniCase::ascii("ANALYZER") => TokenKind::Keyword(Keyword::Analyzer),
UniCase::ascii("AS") => Some(TokenKind::Keyword(Keyword::As)), UniCase::ascii("AS") => TokenKind::Keyword(Keyword::As),
UniCase::ascii("ASCENDING") => Some(TokenKind::Keyword(Keyword::Ascending)), UniCase::ascii("ASCENDING") => TokenKind::Keyword(Keyword::Ascending),
UniCase::ascii("ASC") => Some(TokenKind::Keyword(Keyword::Ascending)), UniCase::ascii("ASC") => TokenKind::Keyword(Keyword::Ascending),
UniCase::ascii("ASCII") => Some(TokenKind::Keyword(Keyword::Ascii)), UniCase::ascii("ASCII") => TokenKind::Keyword(Keyword::Ascii),
UniCase::ascii("ASSERT") => Some(TokenKind::Keyword(Keyword::Assert)), UniCase::ascii("ASSERT") => TokenKind::Keyword(Keyword::Assert),
UniCase::ascii("AT") => Some(TokenKind::Keyword(Keyword::At)), UniCase::ascii("AT") => TokenKind::Keyword(Keyword::At),
UniCase::ascii("BEFORE") => Some(TokenKind::Keyword(Keyword::Before)), UniCase::ascii("BEFORE") => TokenKind::Keyword(Keyword::Before),
UniCase::ascii("BEGIN") => Some(TokenKind::Keyword(Keyword::Begin)), UniCase::ascii("BEGIN") => TokenKind::Keyword(Keyword::Begin),
UniCase::ascii("BLANK") => Some(TokenKind::Keyword(Keyword::Blank)), UniCase::ascii("BLANK") => TokenKind::Keyword(Keyword::Blank),
UniCase::ascii("BM25") => Some(TokenKind::Keyword(Keyword::Bm25)), UniCase::ascii("BM25") => TokenKind::Keyword(Keyword::Bm25),
UniCase::ascii("BREAK") => Some(TokenKind::Keyword(Keyword::Break)), UniCase::ascii("BREAK") => TokenKind::Keyword(Keyword::Break),
UniCase::ascii("BY") => Some(TokenKind::Keyword(Keyword::By)), UniCase::ascii("BY") => TokenKind::Keyword(Keyword::By),
UniCase::ascii("CAMEL") => Some(TokenKind::Keyword(Keyword::Camel)), UniCase::ascii("CAMEL") => TokenKind::Keyword(Keyword::Camel),
UniCase::ascii("CANCEL") => Some(TokenKind::Keyword(Keyword::Cancel)), UniCase::ascii("CANCEL") => TokenKind::Keyword(Keyword::Cancel),
UniCase::ascii("CHANGEFEED") => Some(TokenKind::Keyword(Keyword::ChangeFeed)), UniCase::ascii("CHANGEFEED") => TokenKind::Keyword(Keyword::ChangeFeed),
UniCase::ascii("CHANGES") => Some(TokenKind::Keyword(Keyword::Changes)), UniCase::ascii("CHANGES") => TokenKind::Keyword(Keyword::Changes),
UniCase::ascii("CAPACITY") => Some(TokenKind::Keyword(Keyword::Capacity)), UniCase::ascii("CAPACITY") => TokenKind::Keyword(Keyword::Capacity),
UniCase::ascii("CLASS") => Some(TokenKind::Keyword(Keyword::Class)), UniCase::ascii("CLASS") => TokenKind::Keyword(Keyword::Class),
UniCase::ascii("COMMENT") => Some(TokenKind::Keyword(Keyword::Comment)), UniCase::ascii("COMMENT") => TokenKind::Keyword(Keyword::Comment),
UniCase::ascii("COMMIT") => Some(TokenKind::Keyword(Keyword::Commit)), UniCase::ascii("COMMIT") => TokenKind::Keyword(Keyword::Commit),
UniCase::ascii("CONTENT") => Some(TokenKind::Keyword(Keyword::Content)), UniCase::ascii("CONTENT") => TokenKind::Keyword(Keyword::Content),
UniCase::ascii("CONTINUE") => Some(TokenKind::Keyword(Keyword::Continue)), UniCase::ascii("CONTINUE") => TokenKind::Keyword(Keyword::Continue),
UniCase::ascii("CREATE") => Some(TokenKind::Keyword(Keyword::Create)), UniCase::ascii("CREATE") => TokenKind::Keyword(Keyword::Create),
UniCase::ascii("DATABASE") => Some(TokenKind::Keyword(Keyword::Database)), UniCase::ascii("DATABASE") => TokenKind::Keyword(Keyword::Database),
UniCase::ascii("DB") => Some(TokenKind::Keyword(Keyword::Database)), UniCase::ascii("DB") => TokenKind::Keyword(Keyword::Database),
UniCase::ascii("DEFAULT") => Some(TokenKind::Keyword(Keyword::Default)), UniCase::ascii("DEFAULT") => TokenKind::Keyword(Keyword::Default),
UniCase::ascii("DEFINE") => Some(TokenKind::Keyword(Keyword::Define)), UniCase::ascii("DEFINE") => TokenKind::Keyword(Keyword::Define),
UniCase::ascii("DELETE") => Some(TokenKind::Keyword(Keyword::Delete)), UniCase::ascii("DELETE") => TokenKind::Keyword(Keyword::Delete),
UniCase::ascii("DESCENDING") => Some(TokenKind::Keyword(Keyword::Descending)), UniCase::ascii("DESCENDING") => TokenKind::Keyword(Keyword::Descending),
UniCase::ascii("DESC") => Some(TokenKind::Keyword(Keyword::Descending)), UniCase::ascii("DESC") => TokenKind::Keyword(Keyword::Descending),
UniCase::ascii("DIFF") => Some(TokenKind::Keyword(Keyword::Diff)), UniCase::ascii("DIFF") => TokenKind::Keyword(Keyword::Diff),
UniCase::ascii("DIMENSION") => Some(TokenKind::Keyword(Keyword::Dimension)), UniCase::ascii("DIMENSION") => TokenKind::Keyword(Keyword::Dimension),
UniCase::ascii("DISTANCE") => Some(TokenKind::Keyword(Keyword::Distance)), UniCase::ascii("DISTANCE") => TokenKind::Keyword(Keyword::Distance),
UniCase::ascii("DIST") => Some(TokenKind::Keyword(Keyword::Distance)), UniCase::ascii("DIST") => TokenKind::Keyword(Keyword::Distance),
UniCase::ascii("DOC_IDS_CACHE") => Some(TokenKind::Keyword(Keyword::DocIdsCache)), UniCase::ascii("DOC_IDS_CACHE") => TokenKind::Keyword(Keyword::DocIdsCache),
UniCase::ascii("DOC_IDS_ORDER") => Some(TokenKind::Keyword(Keyword::DocIdsOrder)), UniCase::ascii("DOC_IDS_ORDER") => TokenKind::Keyword(Keyword::DocIdsOrder),
UniCase::ascii("DOC_LENGTHS_CACHE") => Some(TokenKind::Keyword(Keyword::DocLengthsCache)), UniCase::ascii("DOC_LENGTHS_CACHE") => TokenKind::Keyword(Keyword::DocLengthsCache),
UniCase::ascii("DOC_LENGTHS_ORDER") => Some(TokenKind::Keyword(Keyword::DocLengthsOrder)), UniCase::ascii("DOC_LENGTHS_ORDER") => TokenKind::Keyword(Keyword::DocLengthsOrder),
UniCase::ascii("DROP") => Some(TokenKind::Keyword(Keyword::Drop)), UniCase::ascii("DROP") => TokenKind::Keyword(Keyword::Drop),
UniCase::ascii("DUPLICATE") => Some(TokenKind::Keyword(Keyword::Duplicate)), UniCase::ascii("DUPLICATE") => TokenKind::Keyword(Keyword::Duplicate),
UniCase::ascii("EDGENGRAM") => Some(TokenKind::Keyword(Keyword::Edgengram)), UniCase::ascii("EDGENGRAM") => TokenKind::Keyword(Keyword::Edgengram),
UniCase::ascii("EVENT") => Some(TokenKind::Keyword(Keyword::Event)), UniCase::ascii("EVENT") => TokenKind::Keyword(Keyword::Event),
UniCase::ascii("ELSE") => Some(TokenKind::Keyword(Keyword::Else)), UniCase::ascii("ELSE") => TokenKind::Keyword(Keyword::Else),
UniCase::ascii("END") => Some(TokenKind::Keyword(Keyword::End)), UniCase::ascii("END") => TokenKind::Keyword(Keyword::End),
UniCase::ascii("EXISTS") => Some(TokenKind::Keyword(Keyword::Exists)), UniCase::ascii("EXISTS") => TokenKind::Keyword(Keyword::Exists),
UniCase::ascii("EXPLAIN") => Some(TokenKind::Keyword(Keyword::Explain)), UniCase::ascii("EXPLAIN") => TokenKind::Keyword(Keyword::Explain),
UniCase::ascii("false") => Some(TokenKind::Keyword(Keyword::False)), UniCase::ascii("false") => TokenKind::Keyword(Keyword::False),
UniCase::ascii("FETCH") => Some(TokenKind::Keyword(Keyword::Fetch)), UniCase::ascii("FETCH") => TokenKind::Keyword(Keyword::Fetch),
UniCase::ascii("FIELD") => Some(TokenKind::Keyword(Keyword::Field)), UniCase::ascii("FIELD") => TokenKind::Keyword(Keyword::Field),
UniCase::ascii("FIELDS") => Some(TokenKind::Keyword(Keyword::Fields)), UniCase::ascii("FIELDS") => TokenKind::Keyword(Keyword::Fields),
UniCase::ascii("COLUMNS") => Some(TokenKind::Keyword(Keyword::Fields)), UniCase::ascii("COLUMNS") => TokenKind::Keyword(Keyword::Fields),
UniCase::ascii("FILTERS") => Some(TokenKind::Keyword(Keyword::Filters)), UniCase::ascii("FILTERS") => TokenKind::Keyword(Keyword::Filters),
UniCase::ascii("FLEXIBLE") => Some(TokenKind::Keyword(Keyword::Flexible)), UniCase::ascii("FLEXIBLE") => TokenKind::Keyword(Keyword::Flexible),
UniCase::ascii("FLEXI") => Some(TokenKind::Keyword(Keyword::Flexible)), UniCase::ascii("FLEXI") => TokenKind::Keyword(Keyword::Flexible),
UniCase::ascii("FLEX") => Some(TokenKind::Keyword(Keyword::Flexible)), UniCase::ascii("FLEX") => TokenKind::Keyword(Keyword::Flexible),
UniCase::ascii("FOR") => Some(TokenKind::Keyword(Keyword::For)), UniCase::ascii("FOR") => TokenKind::Keyword(Keyword::For),
UniCase::ascii("FROM") => Some(TokenKind::Keyword(Keyword::From)), UniCase::ascii("FROM") => TokenKind::Keyword(Keyword::From),
UniCase::ascii("FULL") => Some(TokenKind::Keyword(Keyword::Full)), UniCase::ascii("FULL") => TokenKind::Keyword(Keyword::Full),
UniCase::ascii("FUNCTION") => Some(TokenKind::Keyword(Keyword::Function)), UniCase::ascii("FUNCTION") => TokenKind::Keyword(Keyword::Function),
UniCase::ascii("GROUP") => Some(TokenKind::Keyword(Keyword::Group)), UniCase::ascii("GROUP") => TokenKind::Keyword(Keyword::Group),
UniCase::ascii("HIGHLIGHTS") => Some(TokenKind::Keyword(Keyword::Highlights)), UniCase::ascii("HIGHLIGHTS") => TokenKind::Keyword(Keyword::Highlights),
UniCase::ascii("IGNORE") => Some(TokenKind::Keyword(Keyword::Ignore)), UniCase::ascii("IGNORE") => TokenKind::Keyword(Keyword::Ignore),
UniCase::ascii("INDEX") => Some(TokenKind::Keyword(Keyword::Index)), UniCase::ascii("INDEX") => TokenKind::Keyword(Keyword::Index),
UniCase::ascii("INFO") => Some(TokenKind::Keyword(Keyword::Info)), UniCase::ascii("INFO") => TokenKind::Keyword(Keyword::Info),
UniCase::ascii("INSERT") => Some(TokenKind::Keyword(Keyword::Insert)), UniCase::ascii("INSERT") => TokenKind::Keyword(Keyword::Insert),
UniCase::ascii("INTO") => Some(TokenKind::Keyword(Keyword::Into)), UniCase::ascii("INTO") => TokenKind::Keyword(Keyword::Into),
UniCase::ascii("IF") => Some(TokenKind::Keyword(Keyword::If)), UniCase::ascii("IF") => TokenKind::Keyword(Keyword::If),
UniCase::ascii("IS") => Some(TokenKind::Keyword(Keyword::Is)), UniCase::ascii("IS") => TokenKind::Keyword(Keyword::Is),
UniCase::ascii("KEY") => Some(TokenKind::Keyword(Keyword::Key)), UniCase::ascii("KEY") => TokenKind::Keyword(Keyword::Key),
UniCase::ascii("KILL") => Some(TokenKind::Keyword(Keyword::Kill)), UniCase::ascii("KILL") => TokenKind::Keyword(Keyword::Kill),
UniCase::ascii("KNN") => Some(TokenKind::Keyword(Keyword::Knn)), UniCase::ascii("KNN") => TokenKind::Keyword(Keyword::Knn),
UniCase::ascii("LET") => Some(TokenKind::Keyword(Keyword::Let)), UniCase::ascii("LET") => TokenKind::Keyword(Keyword::Let),
UniCase::ascii("LIMIT") => Some(TokenKind::Keyword(Keyword::Limit)), UniCase::ascii("LIMIT") => TokenKind::Keyword(Keyword::Limit),
UniCase::ascii("LIVE") => Some(TokenKind::Keyword(Keyword::Live)), UniCase::ascii("LIVE") => TokenKind::Keyword(Keyword::Live),
UniCase::ascii("LOWERCASE") => Some(TokenKind::Keyword(Keyword::Lowercase)), UniCase::ascii("LOWERCASE") => TokenKind::Keyword(Keyword::Lowercase),
UniCase::ascii("MERGE") => Some(TokenKind::Keyword(Keyword::Merge)), UniCase::ascii("MERGE") => TokenKind::Keyword(Keyword::Merge),
UniCase::ascii("MODEL") => Some(TokenKind::Keyword(Keyword::Model)), UniCase::ascii("MODEL") => TokenKind::Keyword(Keyword::Model),
UniCase::ascii("MTREE") => Some(TokenKind::Keyword(Keyword::MTree)), UniCase::ascii("MTREE") => TokenKind::Keyword(Keyword::MTree),
UniCase::ascii("MTREE_CACHE") => Some(TokenKind::Keyword(Keyword::MTreeCache)), UniCase::ascii("MTREE_CACHE") => TokenKind::Keyword(Keyword::MTreeCache),
UniCase::ascii("NAMESPACE") => Some(TokenKind::Keyword(Keyword::Namespace)), UniCase::ascii("NAMESPACE") => TokenKind::Keyword(Keyword::Namespace),
UniCase::ascii("NS") => Some(TokenKind::Keyword(Keyword::Namespace)), UniCase::ascii("NS") => TokenKind::Keyword(Keyword::Namespace),
UniCase::ascii("NGRAM") => Some(TokenKind::Keyword(Keyword::Ngram)), UniCase::ascii("NGRAM") => TokenKind::Keyword(Keyword::Ngram),
UniCase::ascii("NO") => Some(TokenKind::Keyword(Keyword::No)), UniCase::ascii("NO") => TokenKind::Keyword(Keyword::No),
UniCase::ascii("NOINDEX") => Some(TokenKind::Keyword(Keyword::NoIndex)), UniCase::ascii("NOINDEX") => TokenKind::Keyword(Keyword::NoIndex),
UniCase::ascii("NONE") => Some(TokenKind::Keyword(Keyword::None)), UniCase::ascii("NONE") => TokenKind::Keyword(Keyword::None),
UniCase::ascii("NULL") => Some(TokenKind::Keyword(Keyword::Null)), UniCase::ascii("NULL") => TokenKind::Keyword(Keyword::Null),
UniCase::ascii("NUMERIC") => Some(TokenKind::Keyword(Keyword::Numeric)), UniCase::ascii("NUMERIC") => TokenKind::Keyword(Keyword::Numeric),
UniCase::ascii("OMIT") => Some(TokenKind::Keyword(Keyword::Omit)), UniCase::ascii("OMIT") => TokenKind::Keyword(Keyword::Omit),
UniCase::ascii("ON") => Some(TokenKind::Keyword(Keyword::On)), UniCase::ascii("ON") => TokenKind::Keyword(Keyword::On),
UniCase::ascii("ONLY") => Some(TokenKind::Keyword(Keyword::Only)), UniCase::ascii("ONLY") => TokenKind::Keyword(Keyword::Only),
UniCase::ascii("OPTION") => Some(TokenKind::Keyword(Keyword::Option)), UniCase::ascii("OPTION") => TokenKind::Keyword(Keyword::Option),
UniCase::ascii("ORDER") => Some(TokenKind::Keyword(Keyword::Order)), UniCase::ascii("ORDER") => TokenKind::Keyword(Keyword::Order),
UniCase::ascii("PARALLEL") => Some(TokenKind::Keyword(Keyword::Parallel)), UniCase::ascii("PARALLEL") => TokenKind::Keyword(Keyword::Parallel),
UniCase::ascii("PARAM") => Some(TokenKind::Keyword(Keyword::Param)), UniCase::ascii("PARAM") => TokenKind::Keyword(Keyword::Param),
UniCase::ascii("PASSHASH") => Some(TokenKind::Keyword(Keyword::Passhash)), UniCase::ascii("PASSHASH") => TokenKind::Keyword(Keyword::Passhash),
UniCase::ascii("PASSWORD") => Some(TokenKind::Keyword(Keyword::Password)), UniCase::ascii("PASSWORD") => TokenKind::Keyword(Keyword::Password),
UniCase::ascii("PATCH") => Some(TokenKind::Keyword(Keyword::Patch)), UniCase::ascii("PATCH") => TokenKind::Keyword(Keyword::Patch),
UniCase::ascii("PERMISSIONS") => Some(TokenKind::Keyword(Keyword::Permissions)), UniCase::ascii("PERMISSIONS") => TokenKind::Keyword(Keyword::Permissions),
UniCase::ascii("POSTINGS_CACHE") => Some(TokenKind::Keyword(Keyword::PostingsCache)), UniCase::ascii("POSTINGS_CACHE") => TokenKind::Keyword(Keyword::PostingsCache),
UniCase::ascii("POSTINGS_ORDER") => Some(TokenKind::Keyword(Keyword::PostingsOrder)), UniCase::ascii("POSTINGS_ORDER") => TokenKind::Keyword(Keyword::PostingsOrder),
UniCase::ascii("PUNCT") => Some(TokenKind::Keyword(Keyword::Punct)), UniCase::ascii("PUNCT") => TokenKind::Keyword(Keyword::Punct),
UniCase::ascii("READONLY") => Some(TokenKind::Keyword(Keyword::Readonly)), UniCase::ascii("READONLY") => TokenKind::Keyword(Keyword::Readonly),
UniCase::ascii("RELATE") => Some(TokenKind::Keyword(Keyword::Relate)), UniCase::ascii("RELATE") => TokenKind::Keyword(Keyword::Relate),
UniCase::ascii("REMOVE") => Some(TokenKind::Keyword(Keyword::Remove)), UniCase::ascii("REMOVE") => TokenKind::Keyword(Keyword::Remove),
UniCase::ascii("REPLACE") => Some(TokenKind::Keyword(Keyword::Replace)), UniCase::ascii("REPLACE") => TokenKind::Keyword(Keyword::Replace),
UniCase::ascii("RETURN") => Some(TokenKind::Keyword(Keyword::Return)), UniCase::ascii("RETURN") => TokenKind::Keyword(Keyword::Return),
UniCase::ascii("ROLES") => Some(TokenKind::Keyword(Keyword::Roles)), UniCase::ascii("ROLES") => TokenKind::Keyword(Keyword::Roles),
UniCase::ascii("ROOT") => Some(TokenKind::Keyword(Keyword::Root)), UniCase::ascii("ROOT") => TokenKind::Keyword(Keyword::Root),
UniCase::ascii("KV") => Some(TokenKind::Keyword(Keyword::Root)), UniCase::ascii("KV") => TokenKind::Keyword(Keyword::Root),
UniCase::ascii("SCHEMAFULL") => Some(TokenKind::Keyword(Keyword::Schemafull)), UniCase::ascii("SCHEMAFULL") => TokenKind::Keyword(Keyword::Schemafull),
UniCase::ascii("SCHEMAFUL") => Some(TokenKind::Keyword(Keyword::Schemafull)), UniCase::ascii("SCHEMAFUL") => TokenKind::Keyword(Keyword::Schemafull),
UniCase::ascii("SCHEMALESS") => Some(TokenKind::Keyword(Keyword::Schemaless)), UniCase::ascii("SCHEMALESS") => TokenKind::Keyword(Keyword::Schemaless),
UniCase::ascii("SCOPE") => Some(TokenKind::Keyword(Keyword::Scope)), UniCase::ascii("SCOPE") => TokenKind::Keyword(Keyword::Scope),
UniCase::ascii("SC") => Some(TokenKind::Keyword(Keyword::Scope)), UniCase::ascii("SC") => TokenKind::Keyword(Keyword::Scope),
UniCase::ascii("SEARCH") => Some(TokenKind::Keyword(Keyword::Search)), UniCase::ascii("SEARCH") => TokenKind::Keyword(Keyword::Search),
UniCase::ascii("SELECT") => Some(TokenKind::Keyword(Keyword::Select)), UniCase::ascii("SELECT") => TokenKind::Keyword(Keyword::Select),
UniCase::ascii("SESSION") => Some(TokenKind::Keyword(Keyword::Session)), UniCase::ascii("SESSION") => TokenKind::Keyword(Keyword::Session),
UniCase::ascii("SET") => Some(TokenKind::Keyword(Keyword::Set)), UniCase::ascii("SET") => TokenKind::Keyword(Keyword::Set),
UniCase::ascii("SHOW") => Some(TokenKind::Keyword(Keyword::Show)), UniCase::ascii("SHOW") => TokenKind::Keyword(Keyword::Show),
UniCase::ascii("SIGNIN") => Some(TokenKind::Keyword(Keyword::Signin)), UniCase::ascii("SIGNIN") => TokenKind::Keyword(Keyword::Signin),
UniCase::ascii("SIGNUP") => Some(TokenKind::Keyword(Keyword::Signup)), UniCase::ascii("SIGNUP") => TokenKind::Keyword(Keyword::Signup),
UniCase::ascii("SINCE") => Some(TokenKind::Keyword(Keyword::Since)), UniCase::ascii("SINCE") => TokenKind::Keyword(Keyword::Since),
UniCase::ascii("SLEEP") => Some(TokenKind::Keyword(Keyword::Sleep)), UniCase::ascii("SLEEP") => TokenKind::Keyword(Keyword::Sleep),
UniCase::ascii("SNOWBALL") => Some(TokenKind::Keyword(Keyword::Snowball)), UniCase::ascii("SNOWBALL") => TokenKind::Keyword(Keyword::Snowball),
UniCase::ascii("SPLIT") => Some(TokenKind::Keyword(Keyword::Split)), UniCase::ascii("SPLIT") => TokenKind::Keyword(Keyword::Split),
UniCase::ascii("START") => Some(TokenKind::Keyword(Keyword::Start)), UniCase::ascii("START") => TokenKind::Keyword(Keyword::Start),
UniCase::ascii("TABLE") => Some(TokenKind::Keyword(Keyword::Table)), UniCase::ascii("TABLE") => TokenKind::Keyword(Keyword::Table),
UniCase::ascii("TB") => Some(TokenKind::Keyword(Keyword::Table)), UniCase::ascii("TB") => TokenKind::Keyword(Keyword::Table),
UniCase::ascii("TERMS_CACHE") => Some(TokenKind::Keyword(Keyword::TermsCache)), UniCase::ascii("TERMS_CACHE") => TokenKind::Keyword(Keyword::TermsCache),
UniCase::ascii("TERMS_ORDER") => Some(TokenKind::Keyword(Keyword::TermsOrder)), UniCase::ascii("TERMS_ORDER") => TokenKind::Keyword(Keyword::TermsOrder),
UniCase::ascii("THEN") => Some(TokenKind::Keyword(Keyword::Then)), UniCase::ascii("THEN") => TokenKind::Keyword(Keyword::Then),
UniCase::ascii("THROW") => Some(TokenKind::Keyword(Keyword::Throw)), UniCase::ascii("THROW") => TokenKind::Keyword(Keyword::Throw),
UniCase::ascii("TIMEOUT") => Some(TokenKind::Keyword(Keyword::Timeout)), UniCase::ascii("TIMEOUT") => TokenKind::Keyword(Keyword::Timeout),
UniCase::ascii("TOKENIZERS") => Some(TokenKind::Keyword(Keyword::Tokenizers)), UniCase::ascii("TOKENIZERS") => TokenKind::Keyword(Keyword::Tokenizers),
UniCase::ascii("TOKEN") => Some(TokenKind::Keyword(Keyword::Token)), UniCase::ascii("TOKEN") => TokenKind::Keyword(Keyword::Token),
UniCase::ascii("TRANSACTION") => Some(TokenKind::Keyword(Keyword::Transaction)), UniCase::ascii("TRANSACTION") => TokenKind::Keyword(Keyword::Transaction),
UniCase::ascii("true") => Some(TokenKind::Keyword(Keyword::True)), UniCase::ascii("true") => TokenKind::Keyword(Keyword::True),
UniCase::ascii("TYPE") => Some(TokenKind::Keyword(Keyword::Type)), UniCase::ascii("TYPE") => TokenKind::Keyword(Keyword::Type),
UniCase::ascii("UNIQUE") => Some(TokenKind::Keyword(Keyword::Unique)), UniCase::ascii("UNIQUE") => TokenKind::Keyword(Keyword::Unique),
UniCase::ascii("UNSET") => Some(TokenKind::Keyword(Keyword::Unset)), UniCase::ascii("UNSET") => TokenKind::Keyword(Keyword::Unset),
UniCase::ascii("UPDATE") => Some(TokenKind::Keyword(Keyword::Update)), UniCase::ascii("UPDATE") => TokenKind::Keyword(Keyword::Update),
UniCase::ascii("UPPERCASE") => Some(TokenKind::Keyword(Keyword::Uppercase)), UniCase::ascii("UPPERCASE") => TokenKind::Keyword(Keyword::Uppercase),
UniCase::ascii("USE") => Some(TokenKind::Keyword(Keyword::Use)), UniCase::ascii("USE") => TokenKind::Keyword(Keyword::Use),
UniCase::ascii("USER") => Some(TokenKind::Keyword(Keyword::User)), UniCase::ascii("USER") => TokenKind::Keyword(Keyword::User),
UniCase::ascii("VALUE") => Some(TokenKind::Keyword(Keyword::Value)), UniCase::ascii("VALUE") => TokenKind::Keyword(Keyword::Value),
UniCase::ascii("VALUES") => Some(TokenKind::Keyword(Keyword::Values)), UniCase::ascii("VALUES") => TokenKind::Keyword(Keyword::Values),
UniCase::ascii("VERSION") => Some(TokenKind::Keyword(Keyword::Version)), UniCase::ascii("VERSION") => TokenKind::Keyword(Keyword::Version),
UniCase::ascii("VS") => Some(TokenKind::Keyword(Keyword::Vs)), UniCase::ascii("VS") => TokenKind::Keyword(Keyword::Vs),
UniCase::ascii("WHEN") => Some(TokenKind::Keyword(Keyword::When)), UniCase::ascii("WHEN") => TokenKind::Keyword(Keyword::When),
UniCase::ascii("WHERE") => Some(TokenKind::Keyword(Keyword::Where)), UniCase::ascii("WHERE") => TokenKind::Keyword(Keyword::Where),
UniCase::ascii("WITH") => Some(TokenKind::Keyword(Keyword::With)), UniCase::ascii("WITH") => TokenKind::Keyword(Keyword::With),
UniCase::ascii("ALLINSIDE") => Some(TokenKind::Keyword(Keyword::AllInside)), UniCase::ascii("ALLINSIDE") => TokenKind::Keyword(Keyword::AllInside),
UniCase::ascii("ANDKW") => Some(TokenKind::Keyword(Keyword::AndKw)), UniCase::ascii("ANDKW") => TokenKind::Keyword(Keyword::AndKw),
UniCase::ascii("ANYINSIDE") => Some(TokenKind::Keyword(Keyword::AnyInside)), UniCase::ascii("ANYINSIDE") => TokenKind::Keyword(Keyword::AnyInside),
UniCase::ascii("INSIDE") => Some(TokenKind::Keyword(Keyword::Inside)), UniCase::ascii("INSIDE") => TokenKind::Keyword(Keyword::Inside),
UniCase::ascii("INTERSECTS") => Some(TokenKind::Keyword(Keyword::Intersects)), UniCase::ascii("INTERSECTS") => TokenKind::Keyword(Keyword::Intersects),
UniCase::ascii("NONEINSIDE") => Some(TokenKind::Keyword(Keyword::NoneInside)), UniCase::ascii("NONEINSIDE") => TokenKind::Keyword(Keyword::NoneInside),
UniCase::ascii("NOTINSIDE") => Some(TokenKind::Keyword(Keyword::NotInside)), UniCase::ascii("NOTINSIDE") => TokenKind::Keyword(Keyword::NotInside),
UniCase::ascii("OR") => Some(TokenKind::Keyword(Keyword::OrKw)), UniCase::ascii("OR") => TokenKind::Keyword(Keyword::OrKw),
UniCase::ascii("OUTSIDE") => Some(TokenKind::Keyword(Keyword::Outside)), UniCase::ascii("OUTSIDE") => TokenKind::Keyword(Keyword::Outside),
UniCase::ascii("NOT") => Some(TokenKind::Keyword(Keyword::Not)), UniCase::ascii("NOT") => TokenKind::Keyword(Keyword::Not),
UniCase::ascii("AND") => Some(TokenKind::Keyword(Keyword::And)), UniCase::ascii("AND") => TokenKind::Keyword(Keyword::And),
UniCase::ascii("COLLATE") => Some(TokenKind::Keyword(Keyword::Collate)), UniCase::ascii("COLLATE") => TokenKind::Keyword(Keyword::Collate),
UniCase::ascii("CONTAINSALL") => Some(TokenKind::Keyword(Keyword::ContainsAll)), UniCase::ascii("CONTAINSALL") => TokenKind::Keyword(Keyword::ContainsAll),
UniCase::ascii("CONTAINSANY") => Some(TokenKind::Keyword(Keyword::ContainsAny)), UniCase::ascii("CONTAINSANY") => TokenKind::Keyword(Keyword::ContainsAny),
UniCase::ascii("CONTAINSNONE") => Some(TokenKind::Keyword(Keyword::ContainsNone)), UniCase::ascii("CONTAINSNONE") => TokenKind::Keyword(Keyword::ContainsNone),
UniCase::ascii("CONTAINSNOT") => Some(TokenKind::Keyword(Keyword::ContainsNot)), UniCase::ascii("CONTAINSNOT") => TokenKind::Keyword(Keyword::ContainsNot),
UniCase::ascii("CONTAINS") => Some(TokenKind::Keyword(Keyword::Contains)), UniCase::ascii("CONTAINS") => TokenKind::Keyword(Keyword::Contains),
UniCase::ascii("IN") => Some(TokenKind::Keyword(Keyword::In)), UniCase::ascii("IN") => TokenKind::Keyword(Keyword::In),
UniCase::ascii("ANY") => Some(TokenKind::Keyword(Keyword::Any)), UniCase::ascii("ANY") => TokenKind::Keyword(Keyword::Any),
UniCase::ascii("ARRAY") => Some(TokenKind::Keyword(Keyword::Array)), UniCase::ascii("ARRAY") => TokenKind::Keyword(Keyword::Array),
UniCase::ascii("GEOMETRY") => Some(TokenKind::Keyword(Keyword::Geometry)), UniCase::ascii("GEOMETRY") => TokenKind::Keyword(Keyword::Geometry),
UniCase::ascii("RECORD") => Some(TokenKind::Keyword(Keyword::Record)), UniCase::ascii("RECORD") => TokenKind::Keyword(Keyword::Record),
UniCase::ascii("FUTURE") => Some(TokenKind::Keyword(Keyword::Future)), UniCase::ascii("FUTURE") => TokenKind::Keyword(Keyword::Future),
UniCase::ascii("BOOL") => Some(TokenKind::Keyword(Keyword::Bool)), UniCase::ascii("BOOL") => TokenKind::Keyword(Keyword::Bool),
UniCase::ascii("BYTES") => Some(TokenKind::Keyword(Keyword::Bytes)), UniCase::ascii("BYTES") => TokenKind::Keyword(Keyword::Bytes),
UniCase::ascii("DATETIME") => Some(TokenKind::Keyword(Keyword::Datetime)), UniCase::ascii("DATETIME") => TokenKind::Keyword(Keyword::Datetime),
UniCase::ascii("DECIMAL") => Some(TokenKind::Keyword(Keyword::Decimal)), UniCase::ascii("DECIMAL") => TokenKind::Keyword(Keyword::Decimal),
UniCase::ascii("DURATION") => Some(TokenKind::Keyword(Keyword::Duration)), UniCase::ascii("DURATION") => TokenKind::Keyword(Keyword::Duration),
UniCase::ascii("FLOAT") => Some(TokenKind::Keyword(Keyword::Float)), UniCase::ascii("FLOAT") => TokenKind::Keyword(Keyword::Float),
UniCase::ascii("fn") => Some(TokenKind::Keyword(Keyword::Fn)), UniCase::ascii("fn") => TokenKind::Keyword(Keyword::Fn),
UniCase::ascii("ml") => Some(TokenKind::Keyword(Keyword::ML)), UniCase::ascii("ml") => TokenKind::Keyword(Keyword::ML),
UniCase::ascii("INT") => Some(TokenKind::Keyword(Keyword::Int)), UniCase::ascii("INT") => TokenKind::Keyword(Keyword::Int),
UniCase::ascii("NUMBER") => Some(TokenKind::Keyword(Keyword::Number)), UniCase::ascii("NUMBER") => TokenKind::Keyword(Keyword::Number),
UniCase::ascii("OBJECT") => Some(TokenKind::Keyword(Keyword::Object)), UniCase::ascii("OBJECT") => TokenKind::Keyword(Keyword::Object),
UniCase::ascii("STRING") => Some(TokenKind::Keyword(Keyword::String)), UniCase::ascii("STRING") => TokenKind::Keyword(Keyword::String),
UniCase::ascii("UUID") => Some(TokenKind::Keyword(Keyword::Uuid)), UniCase::ascii("UUID") => TokenKind::Keyword(Keyword::Uuid),
UniCase::ascii("ULID") => Some(TokenKind::Keyword(Keyword::Ulid)), UniCase::ascii("ULID") => TokenKind::Keyword(Keyword::Ulid),
UniCase::ascii("RAND") => Some(TokenKind::Keyword(Keyword::Rand)), UniCase::ascii("RAND") => TokenKind::Keyword(Keyword::Rand),
UniCase::ascii("FEATURE") => Some(TokenKind::Keyword(Keyword::Feature)), UniCase::ascii("FEATURE") => TokenKind::Keyword(Keyword::Feature),
UniCase::ascii("LINE") => Some(TokenKind::Keyword(Keyword::Line)), UniCase::ascii("LINE") => TokenKind::Keyword(Keyword::Line),
UniCase::ascii("POINT") => Some(TokenKind::Keyword(Keyword::Point)), UniCase::ascii("POINT") => TokenKind::Keyword(Keyword::Point),
UniCase::ascii("POLYGON") => Some(TokenKind::Keyword(Keyword::Polygon)), UniCase::ascii("POLYGON") => TokenKind::Keyword(Keyword::Polygon),
UniCase::ascii("MULTIPOINT") => Some(TokenKind::Keyword(Keyword::MultiPoint)), UniCase::ascii("MULTIPOINT") => TokenKind::Keyword(Keyword::MultiPoint),
UniCase::ascii("MULTILINE") => Some(TokenKind::Keyword(Keyword::MultiLine)), UniCase::ascii("MULTILINE") => TokenKind::Keyword(Keyword::MultiLine),
UniCase::ascii("MULTIPOLYGON") => Some(TokenKind::Keyword(Keyword::MultiPolygon)), UniCase::ascii("MULTIPOLYGON") => TokenKind::Keyword(Keyword::MultiPolygon),
UniCase::ascii("COLLECTION") => Some(TokenKind::Keyword(Keyword::Collection)), UniCase::ascii("COLLECTION") => TokenKind::Keyword(Keyword::Collection),
// Languages // Languages
UniCase::ascii("ARABIC") => Some(TokenKind::Language(Language::Arabic)), UniCase::ascii("ARABIC") => TokenKind::Language(Language::Arabic),
UniCase::ascii("ARA") => Some(TokenKind::Language(Language::Arabic)), UniCase::ascii("ARA") => TokenKind::Language(Language::Arabic),
UniCase::ascii("AR") => Some(TokenKind::Language(Language::Arabic)), UniCase::ascii("AR") => TokenKind::Language(Language::Arabic),
UniCase::ascii("DANISH") => Some(TokenKind::Language(Language::Danish)), UniCase::ascii("DANISH") => TokenKind::Language(Language::Danish),
UniCase::ascii("DAN") => Some(TokenKind::Language(Language::Danish)), UniCase::ascii("DAN") => TokenKind::Language(Language::Danish),
UniCase::ascii("DA") => Some(TokenKind::Language(Language::Danish)), UniCase::ascii("DA") => TokenKind::Language(Language::Danish),
UniCase::ascii("DUTCH") => Some(TokenKind::Language(Language::Dutch)), UniCase::ascii("DUTCH") => TokenKind::Language(Language::Dutch),
UniCase::ascii("NLD") => Some(TokenKind::Language(Language::Dutch)), UniCase::ascii("NLD") => TokenKind::Language(Language::Dutch),
UniCase::ascii("NL") => Some(TokenKind::Language(Language::Dutch)), UniCase::ascii("NL") => TokenKind::Language(Language::Dutch),
UniCase::ascii("ENGLISH") => Some(TokenKind::Language(Language::English)), UniCase::ascii("ENGLISH") => TokenKind::Language(Language::English),
UniCase::ascii("ENG") => Some(TokenKind::Language(Language::English)), UniCase::ascii("ENG") => TokenKind::Language(Language::English),
UniCase::ascii("EN") => Some(TokenKind::Language(Language::English)), UniCase::ascii("EN") => TokenKind::Language(Language::English),
UniCase::ascii("FRENCH") => Some(TokenKind::Language(Language::French)), UniCase::ascii("FRENCH") => TokenKind::Language(Language::French),
UniCase::ascii("FRA") => Some(TokenKind::Language(Language::French)), UniCase::ascii("FRA") => TokenKind::Language(Language::French),
UniCase::ascii("FR") => Some(TokenKind::Language(Language::French)), UniCase::ascii("FR") => TokenKind::Language(Language::French),
UniCase::ascii("GERMAN") => Some(TokenKind::Language(Language::German)), UniCase::ascii("GERMAN") => TokenKind::Language(Language::German),
UniCase::ascii("DEU") => Some(TokenKind::Language(Language::German)), UniCase::ascii("DEU") => TokenKind::Language(Language::German),
UniCase::ascii("DE") => Some(TokenKind::Language(Language::German)), UniCase::ascii("DE") => TokenKind::Language(Language::German),
UniCase::ascii("GREEK") => Some(TokenKind::Language(Language::Greek)), UniCase::ascii("GREEK") => TokenKind::Language(Language::Greek),
UniCase::ascii("ELL") => Some(TokenKind::Language(Language::Greek)), UniCase::ascii("ELL") => TokenKind::Language(Language::Greek),
UniCase::ascii("EL") => Some(TokenKind::Language(Language::Greek)), UniCase::ascii("EL") => TokenKind::Language(Language::Greek),
UniCase::ascii("HUNGARIAN") => Some(TokenKind::Language(Language::Hungarian)), UniCase::ascii("HUNGARIAN") => TokenKind::Language(Language::Hungarian),
UniCase::ascii("HUN") => Some(TokenKind::Language(Language::Hungarian)), UniCase::ascii("HUN") => TokenKind::Language(Language::Hungarian),
UniCase::ascii("HU") => Some(TokenKind::Language(Language::Hungarian)), UniCase::ascii("HU") => TokenKind::Language(Language::Hungarian),
UniCase::ascii("ITALIAN") => Some(TokenKind::Language(Language::Italian)), UniCase::ascii("ITALIAN") => TokenKind::Language(Language::Italian),
UniCase::ascii("ITA") => Some(TokenKind::Language(Language::Italian)), UniCase::ascii("ITA") => TokenKind::Language(Language::Italian),
UniCase::ascii("IT") => Some(TokenKind::Language(Language::Italian)), UniCase::ascii("IT") => TokenKind::Language(Language::Italian),
UniCase::ascii("NORWEGIAN") => Some(TokenKind::Language(Language::Norwegian)), UniCase::ascii("NORWEGIAN") => TokenKind::Language(Language::Norwegian),
UniCase::ascii("NOR") => Some(TokenKind::Language(Language::Norwegian)), UniCase::ascii("NOR") => TokenKind::Language(Language::Norwegian),
UniCase::ascii("PORTUGUESE") => Some(TokenKind::Language(Language::Portuguese)), UniCase::ascii("PORTUGUESE") => TokenKind::Language(Language::Portuguese),
UniCase::ascii("POR") => Some(TokenKind::Language(Language::Portuguese)), UniCase::ascii("POR") => TokenKind::Language(Language::Portuguese),
UniCase::ascii("PT") => Some(TokenKind::Language(Language::Portuguese)), UniCase::ascii("PT") => TokenKind::Language(Language::Portuguese),
UniCase::ascii("ROMANIAN") => Some(TokenKind::Language(Language::Romanian)), UniCase::ascii("ROMANIAN") => TokenKind::Language(Language::Romanian),
UniCase::ascii("RON") => Some(TokenKind::Language(Language::Romanian)), UniCase::ascii("RON") => TokenKind::Language(Language::Romanian),
UniCase::ascii("RO") => Some(TokenKind::Language(Language::Romanian)), UniCase::ascii("RO") => TokenKind::Language(Language::Romanian),
UniCase::ascii("RUSSIAN") => Some(TokenKind::Language(Language::Russian)), UniCase::ascii("RUSSIAN") => TokenKind::Language(Language::Russian),
UniCase::ascii("RUS") => Some(TokenKind::Language(Language::Russian)), UniCase::ascii("RUS") => TokenKind::Language(Language::Russian),
UniCase::ascii("RU") => Some(TokenKind::Language(Language::Russian)), UniCase::ascii("RU") => TokenKind::Language(Language::Russian),
UniCase::ascii("SPANISH") => Some(TokenKind::Language(Language::Spanish)), UniCase::ascii("SPANISH") => TokenKind::Language(Language::Spanish),
UniCase::ascii("SPA") => Some(TokenKind::Language(Language::Spanish)), UniCase::ascii("SPA") => TokenKind::Language(Language::Spanish),
UniCase::ascii("ES") => Some(TokenKind::Language(Language::Spanish)), UniCase::ascii("ES") => TokenKind::Language(Language::Spanish),
UniCase::ascii("SWEDISH") => Some(TokenKind::Language(Language::Swedish)), UniCase::ascii("SWEDISH") => TokenKind::Language(Language::Swedish),
UniCase::ascii("SWE") => Some(TokenKind::Language(Language::Swedish)), UniCase::ascii("SWE") => TokenKind::Language(Language::Swedish),
UniCase::ascii("SV") => Some(TokenKind::Language(Language::Swedish)), UniCase::ascii("SV") => TokenKind::Language(Language::Swedish),
UniCase::ascii("TAMIL") => Some(TokenKind::Language(Language::Tamil)), UniCase::ascii("TAMIL") => TokenKind::Language(Language::Tamil),
UniCase::ascii("TAM") => Some(TokenKind::Language(Language::Tamil)), UniCase::ascii("TAM") => TokenKind::Language(Language::Tamil),
UniCase::ascii("TA") => Some(TokenKind::Language(Language::Tamil)), UniCase::ascii("TA") => TokenKind::Language(Language::Tamil),
UniCase::ascii("TURKISH") => Some(TokenKind::Language(Language::Turkish)), UniCase::ascii("TURKISH") => TokenKind::Language(Language::Turkish),
UniCase::ascii("TUR") => Some(TokenKind::Language(Language::Turkish)), UniCase::ascii("TUR") => TokenKind::Language(Language::Turkish),
UniCase::ascii("TR") => Some(TokenKind::Language(Language::Turkish)), UniCase::ascii("TR") => TokenKind::Language(Language::Turkish),
// Algorithms // Algorithms
UniCase::ascii("EDDSA") => Some(TokenKind::Algorithm(Algorithm::EdDSA)), UniCase::ascii("EDDSA") => TokenKind::Algorithm(Algorithm::EdDSA),
UniCase::ascii("ES256") => Some(TokenKind::Algorithm(Algorithm::Es256)), UniCase::ascii("ES256") => TokenKind::Algorithm(Algorithm::Es256),
UniCase::ascii("ES384") => Some(TokenKind::Algorithm(Algorithm::Es384)), UniCase::ascii("ES384") => TokenKind::Algorithm(Algorithm::Es384),
UniCase::ascii("ES512") => Some(TokenKind::Algorithm(Algorithm::Es512)), UniCase::ascii("ES512") => TokenKind::Algorithm(Algorithm::Es512),
UniCase::ascii("HS256") => Some(TokenKind::Algorithm(Algorithm::Hs256)), UniCase::ascii("HS256") => TokenKind::Algorithm(Algorithm::Hs256),
UniCase::ascii("HS384") => Some(TokenKind::Algorithm(Algorithm::Hs384)), UniCase::ascii("HS384") => TokenKind::Algorithm(Algorithm::Hs384),
UniCase::ascii("HS512") => Some(TokenKind::Algorithm(Algorithm::Hs512)), UniCase::ascii("HS512") => TokenKind::Algorithm(Algorithm::Hs512),
UniCase::ascii("PS256") => Some(TokenKind::Algorithm(Algorithm::Ps256)), UniCase::ascii("PS256") => TokenKind::Algorithm(Algorithm::Ps256),
UniCase::ascii("PS384") => Some(TokenKind::Algorithm(Algorithm::Ps384)), UniCase::ascii("PS384") => TokenKind::Algorithm(Algorithm::Ps384),
UniCase::ascii("PS512") => Some(TokenKind::Algorithm(Algorithm::Ps512)), UniCase::ascii("PS512") => TokenKind::Algorithm(Algorithm::Ps512),
UniCase::ascii("RS256") => Some(TokenKind::Algorithm(Algorithm::Rs256)), UniCase::ascii("RS256") => TokenKind::Algorithm(Algorithm::Rs256),
UniCase::ascii("RS384") => Some(TokenKind::Algorithm(Algorithm::Rs384)), UniCase::ascii("RS384") => TokenKind::Algorithm(Algorithm::Rs384),
UniCase::ascii("RS512") => Some(TokenKind::Algorithm(Algorithm::Rs512)), UniCase::ascii("RS512") => TokenKind::Algorithm(Algorithm::Rs512),
UniCase::ascii("JWKS") => jwks_token_kind(), // Necessary because `phf_map!` doesn't support `cfg` attributes UniCase::ascii("JWKS") => jwks_token_kind(), // Necessary because `phf_map!` doesn't support `cfg` attributes
// Distance // Distance
UniCase::ascii("EUCLIDEAN") => Some(TokenKind::Distance(DistanceKind::Euclidean)), UniCase::ascii("EUCLIDEAN") => TokenKind::Distance(DistanceKind::Euclidean),
UniCase::ascii("MANHATTAN") => Some(TokenKind::Distance(DistanceKind::Manhattan)), UniCase::ascii("MANHATTAN") => TokenKind::Distance(DistanceKind::Manhattan),
UniCase::ascii("HAMMING") => Some(TokenKind::Distance(DistanceKind::Hamming)), UniCase::ascii("HAMMING") => TokenKind::Distance(DistanceKind::Hamming),
UniCase::ascii("MINKOWSKI") => Some(TokenKind::Distance(DistanceKind::Minkowski)), UniCase::ascii("MINKOWSKI") => TokenKind::Distance(DistanceKind::Minkowski),
}; };
const fn jwks_token_kind() -> Option<TokenKind> { const fn jwks_token_kind() -> TokenKind {
#[cfg(feature = "jwks")] #[cfg(feature = "jwks")]
let token = Some(TokenKind::Algorithm(Algorithm::Jwks)); let token = TokenKind::Algorithm(Algorithm::Jwks);
#[cfg(not(feature = "jwks"))] #[cfg(not(feature = "jwks"))]
let token = None; let token = TokenKind::Identifier;
token token
} }

40
core/src/syn/v2/parser/basic.rs
View file

@ -140,7 +140,32 @@ impl TokenValue for f32 {
})?; })?;
Ok(number) Ok(number)
} }
x => unexpected!(parser, x, "an floating point"), x => unexpected!(parser, x, "a floating point number"),
}
}
}
impl TokenValue for f64 {
fn from_token(parser: &mut Parser<'_>, token: Token) -> ParseResult<Self> {
match token.kind {
TokenKind::Number(NumberKind::NaN) => Ok(f64::NAN),
TokenKind::Number(
NumberKind::Integer
| NumberKind::Float
| NumberKind::Mantissa
| NumberKind::MantissaExponent,
) => {
let number = parser.lexer.string.take().unwrap().parse().map_err(|e| {
ParseError::new(
ParseErrorKind::InvalidFloat {
error: e,
},
token.span,
)
})?;
Ok(number)
}
x => unexpected!(parser, x, "a floating point number"),
} }
} }
} }
@ -283,6 +308,19 @@ impl Parser<'_> {
res res
} }
pub fn parse_signed_float(&mut self) -> ParseResult<f64> {
let neg = self.eat(t!("-"));
if !neg {
self.eat(t!("+"));
}
let res: f64 = self.next_token_value()?;
if neg {
Ok(-res)
} else {
Ok(res)
}
}
/// Parse a token value from the given token. /// Parse a token value from the given token.
pub fn token_value<V: TokenValue>(&mut self, token: Token) -> ParseResult<V> { pub fn token_value<V: TokenValue>(&mut self, token: Token) -> ParseResult<V> {
V::from_token(self, token) V::from_token(self, token)

112
core/src/syn/v2/parser/error.rs
View file

@ -18,6 +18,13 @@ pub enum IntErrorKind {
IntegerOverflow, IntegerOverflow,
} }
#[derive(Debug)]
pub enum MissingKind {
Group,
Split,
Order,
}
#[derive(Debug)] #[derive(Debug)]
pub enum ParseErrorKind { pub enum ParseErrorKind {
/// The parser encountered an unexpected token. /// The parser encountered an unexpected token.
@ -51,17 +58,23 @@ pub enum ParseErrorKind {
InvalidDecimal { InvalidDecimal {
error: rust_decimal::Error, error: rust_decimal::Error,
}, },
DisallowedStatement, DisallowedStatement {
found: TokenKind,
expected: TokenKind,
disallowed: Span,
},
/// The parser encountered an token which could not be lexed correctly. /// The parser encountered an token which could not be lexed correctly.
InvalidToken(LexError), InvalidToken(LexError),
/// Matched a path which was invalid. /// Matched a path which was invalid.
InvalidPath { InvalidPath {
possibly: Option<&'static str>, possibly: Option<&'static str>,
}, },
MissingField {
field: Span,
idiom: String,
kind: MissingKind,
},
NoWhitespace, NoWhitespace,
/// A path in the parser which was not yet finished.
/// Should eventually be removed.
Todo,
} }
/// A parsing error. /// A parsing error.
@ -79,16 +92,19 @@ impl ParseError {
at, at,
} }
} }
pub fn render_on(&self, source: &str) -> RenderedError {
Self::render_on_inner(source, &self.kind, self.at)
}
/// Create a rendered error from the string this error was generated from. /// Create a rendered error from the string this error was generated from.
pub fn render_on(&self, source: &str) -> RenderedError { pub fn render_on_inner(source: &str, kind: &ParseErrorKind, at: Span) -> RenderedError {
match &self.kind { match &kind {
ParseErrorKind::Unexpected { ParseErrorKind::Unexpected {
found, found,
expected, expected,
} => { } => {
let text = format!("Unexpected token '{}' expected {}", found.as_str(), expected); let text = format!("Unexpected token '{}' expected {}", found.as_str(), expected);
let locations = Location::range_of_span(source, self.at); let locations = Location::range_of_span(source, at);
let snippet = Snippet::from_source_location_range(source, locations, None); let snippet = Snippet::from_source_location_range(source, locations, None);
RenderedError { RenderedError {
text, text,
@ -101,7 +117,7 @@ impl ParseError {
explain, explain,
} => { } => {
let text = format!("Unexpected token '{}' expected {}", found.as_str(), expected); let text = format!("Unexpected token '{}' expected {}", found.as_str(), expected);
let locations = Location::range_of_span(source, self.at); let locations = Location::range_of_span(source, at);
let snippet = Snippet::from_source_location_range(source, locations, Some(explain)); let snippet = Snippet::from_source_location_range(source, locations, Some(explain));
RenderedError { RenderedError {
text, text,
@ -112,7 +128,7 @@ impl ParseError {
expected, expected,
} => { } => {
let text = format!("Query ended early, expected {}", expected); let text = format!("Query ended early, expected {}", expected);
let locations = Location::range_of_span(source, self.at); let locations = Location::range_of_span(source, at);
let snippet = Snippet::from_source_location_range(source, locations, None); let snippet = Snippet::from_source_location_range(source, locations, None);
RenderedError { RenderedError {
text, text,
@ -123,8 +139,8 @@ impl ParseError {
expected, expected,
should_close, should_close,
} => { } => {
let text = format!("Expected closing delimiter {}", expected.as_str()); let text = format!("Expected closing delimiter '{}'", expected.as_str());
let locations = Location::range_of_span(source, self.at); let locations = Location::range_of_span(source, at);
let snippet = Snippet::from_source_location_range(source, locations, None); let snippet = Snippet::from_source_location_range(source, locations, None);
let locations = Location::range_of_span(source, *should_close); let locations = Location::range_of_span(source, *should_close);
let close_snippet = Snippet::from_source_location_range( let close_snippet = Snippet::from_source_location_range(
@ -137,27 +153,32 @@ impl ParseError {
snippets: vec![snippet, close_snippet], snippets: vec![snippet, close_snippet],
} }
} }
ParseErrorKind::DisallowedStatement => { ParseErrorKind::DisallowedStatement {
let text = "This statement is not allowed in this location".to_owned(); found,
let locations = Location::range_of_span(source, self.at); expected,
disallowed,
} => {
let text = format!(
"Unexpected token '{}' expected '{}'",
found.as_str(),
expected.as_str()
);
let locations = Location::range_of_span(source, at);
let snippet = Snippet::from_source_location_range(source, locations, None); let snippet = Snippet::from_source_location_range(source, locations, None);
let locations = Location::range_of_span(source, *disallowed);
let dissallowed_snippet = Snippet::from_source_location_range(
source,
locations,
Some("this keyword is not allowed to start a statement in this position"),
);
RenderedError { RenderedError {
text, text,
snippets: vec![snippet], snippets: vec![snippet, dissallowed_snippet],
} }
} }
ParseErrorKind::InvalidToken(e) => { ParseErrorKind::InvalidToken(e) => {
let text = e.to_string(); let text = e.to_string();
let locations = Location::range_of_span(source, self.at); let locations = Location::range_of_span(source, at);
let snippet = Snippet::from_source_location_range(source, locations, None);
RenderedError {
text,
snippets: vec![snippet],
}
}
ParseErrorKind::Todo => {
let text = "Parser hit not yet implemented path".to_string();
let locations = Location::range_of_span(source, self.at);
let snippet = Snippet::from_source_location_range(source, locations, None); let snippet = Snippet::from_source_location_range(source, locations, None);
RenderedError { RenderedError {
text, text,
@ -167,12 +188,12 @@ impl ParseError {
ParseErrorKind::InvalidPath { ParseErrorKind::InvalidPath {
possibly, possibly,
} => { } => {
let mut text = "Invalid path".to_owned(); let mut text = "Invalid function path".to_owned();
if let Some(p) = possibly { if let Some(p) = possibly {
// writing into a string never causes an error. // writing into a string never causes an error.
write!(text, ", did you maybe mean `{}`", p).unwrap(); write!(text, ", did you maybe mean `{}`", p).unwrap();
} }
let locations = Location::range_of_span(source, self.at); let locations = Location::range_of_span(source, at);
let snippet = Snippet::from_source_location_range( let snippet = Snippet::from_source_location_range(
source, source,
locations, locations,
@ -187,7 +208,7 @@ impl ParseError {
ref error, ref error,
} => { } => {
let text = format!("failed to parse integer, {error}"); let text = format!("failed to parse integer, {error}");
let locations = Location::range_of_span(source, self.at); let locations = Location::range_of_span(source, at);
let snippet = Snippet::from_source_location_range(source, locations, None); let snippet = Snippet::from_source_location_range(source, locations, None);
RenderedError { RenderedError {
text: text.to_string(), text: text.to_string(),
@ -198,7 +219,7 @@ impl ParseError {
ref error, ref error,
} => { } => {
let text = format!("failed to parse floating point, {error}"); let text = format!("failed to parse floating point, {error}");
let locations = Location::range_of_span(source, self.at); let locations = Location::range_of_span(source, at);
let snippet = Snippet::from_source_location_range(source, locations, None); let snippet = Snippet::from_source_location_range(source, locations, None);
RenderedError { RenderedError {
text: text.to_string(), text: text.to_string(),
@ -209,7 +230,7 @@ impl ParseError {
ref error, ref error,
} => { } => {
let text = format!("failed to parse decimal number, {error}"); let text = format!("failed to parse decimal number, {error}");
let locations = Location::range_of_span(source, self.at); let locations = Location::range_of_span(source, at);
let snippet = Snippet::from_source_location_range(source, locations, None); let snippet = Snippet::from_source_location_range(source, locations, None);
RenderedError { RenderedError {
text: text.to_string(), text: text.to_string(),
@ -218,13 +239,42 @@ impl ParseError {
} }
ParseErrorKind::NoWhitespace => { ParseErrorKind::NoWhitespace => {
let text = "Whitespace is dissallowed in this position"; let text = "Whitespace is dissallowed in this position";
let locations = Location::range_of_span(source, self.at); let locations = Location::range_of_span(source, at);
let snippet = Snippet::from_source_location_range(source, locations, None); let snippet = Snippet::from_source_location_range(source, locations, None);
RenderedError { RenderedError {
text: text.to_string(), text: text.to_string(),
snippets: vec![snippet], snippets: vec![snippet],
} }
} }
ParseErrorKind::MissingField {
field,
idiom,
kind,
} => {
let text = match kind {
MissingKind::Group => {
format!("Missing group idiom `{idiom}` in statement selection")
}
MissingKind::Split => {
format!("Missing split idiom `{idiom}` in statement selection")
}
MissingKind::Order => {
format!("Missing order idiom `{idiom}` in statement selection")
}
};
let locations = Location::range_of_span(source, at);
let snippet_error = Snippet::from_source_location_range(source, locations, None);
let locations = Location::range_of_span(source, *field);
let snippet_hint = Snippet::from_source_location_range(
source,
locations,
Some("Idiom missing here"),
);
RenderedError {
text: text.to_string(),
snippets: vec![snippet_error, snippet_hint],
}
}
} }
} }
} }

22
core/src/syn/v2/parser/expression.rs
View file

@ -1,8 +1,10 @@
//! This module defines the pratt parser for operators. //! This module defines the pratt parser for operators.
use super::mac::unexpected; use super::mac::unexpected;
use super::ParseError;
use crate::sql::{value::TryNeg, Cast, Expression, Number, Operator, Value}; use crate::sql::{value::TryNeg, Cast, Expression, Number, Operator, Value};
use crate::syn::v2::{ use crate::syn::v2::{
parser::{mac::expected, ParseResult, Parser}, parser::{mac::expected, ParseErrorKind, ParseResult, Parser},
token::{t, NumberKind, TokenKind}, token::{t, NumberKind, TokenKind},
}; };
use std::cmp::Ordering; use std::cmp::Ordering;
@ -60,9 +62,8 @@ impl Parser<'_> {
fn infix_binding_power(token: TokenKind) -> Option<(u8, u8)> { fn infix_binding_power(token: TokenKind) -> Option<(u8, u8)> {
// TODO: Look at ordering of operators. // TODO: Look at ordering of operators.
match token { match token {
// assigment operators have the lowes binding power. // assigment operators have the lowest binding power.
t!("+=") | t!("-=") | t!("+?=") => Some((2, 1)), //t!("+=") | t!("-=") | t!("+?=") => Some((2, 1)),
t!("||") | t!("OR") => Some((3, 4)), t!("||") | t!("OR") => Some((3, 4)),
t!("&&") | t!("AND") => Some((5, 6)), t!("&&") | t!("AND") => Some((5, 6)),
@ -261,7 +262,7 @@ impl Parser<'_> {
} }
// should be unreachable as we previously check if the token was a prefix op. // should be unreachable as we previously check if the token was a prefix op.
_ => unreachable!(), x => unreachable!("found non-operator token {x:?}"),
}; };
let rhs = self.pratt_parse_expr(min_bp)?; let rhs = self.pratt_parse_expr(min_bp)?;
Ok(Value::Expression(Box::new(Expression::Binary { Ok(Value::Expression(Box::new(Expression::Binary {
@ -284,6 +285,17 @@ impl Parser<'_> {
loop { loop {
let token = self.peek(); let token = self.peek();
let Some((l_bp, r_bp)) = Self::infix_binding_power(token.kind) else { let Some((l_bp, r_bp)) = Self::infix_binding_power(token.kind) else {
// explain that assignment operators can't be used in normal expressions.
if let t!("+=") | t!("*=") | t!("-=") | t!("+?=") = token.kind {
return Err(ParseError::new(
ParseErrorKind::UnexpectedExplain {
found: token.kind,
expected: "an operator",
explain: "assignement operator are only allowed in SET and DUPLICATE KEY UPDATE statements",
},
token.span,
));
}
break; break;
}; };

8
core/src/syn/v2/parser/mac.rs
View file

@ -6,7 +6,7 @@ macro_rules! unexpected {
let error = $parser.lexer.error.take().unwrap(); let error = $parser.lexer.error.take().unwrap();
return Err($crate::syn::v2::parser::ParseError::new( return Err($crate::syn::v2::parser::ParseError::new(
$crate::syn::v2::parser::ParseErrorKind::InvalidToken(error), $crate::syn::v2::parser::ParseErrorKind::InvalidToken(error),
$parser.last_span(), $parser.recent_span(),
)); ));
} }
$crate::syn::v2::token::TokenKind::Eof => { $crate::syn::v2::token::TokenKind::Eof => {
@ -15,7 +15,7 @@ macro_rules! unexpected {
$crate::syn::v2::parser::ParseErrorKind::UnexpectedEof { $crate::syn::v2::parser::ParseErrorKind::UnexpectedEof {
expected, expected,
}, },
$parser.last_span(), $parser.recent_span(),
)); ));
} }
x => { x => {
@ -25,7 +25,7 @@ macro_rules! unexpected {
found: x, found: x,
expected, expected,
}, },
$parser.last_span(), $parser.recent_span(),
)); ));
} }
} }
@ -42,7 +42,7 @@ macro_rules! expected {
let error = $parser.lexer.error.take().unwrap(); let error = $parser.lexer.error.take().unwrap();
return Err($crate::syn::v2::parser::ParseError::new( return Err($crate::syn::v2::parser::ParseError::new(
$crate::syn::v2::parser::ParseErrorKind::InvalidToken(error), $crate::syn::v2::parser::ParseErrorKind::InvalidToken(error),
$parser.last_span(), $parser.recent_span(),
)); ));
} }
x => { x => {

30
core/src/syn/v2/parser/mod.rs
View file

@ -21,7 +21,6 @@ use crate::{
sql, sql,
syn::v2::{ syn::v2::{
lexer::{Error as LexError, Lexer}, lexer::{Error as LexError, Lexer},
parser::mac::expected,
token::{t, Span, Token, TokenKind}, token::{t, Span, Token, TokenKind},
}, },
}; };
@ -163,10 +162,15 @@ impl<'a> Parser<'a> {
/// Returns the span of the next token if it was already peeked, otherwise returns the token of /// Returns the span of the next token if it was already peeked, otherwise returns the token of
/// the last consumed token. /// the last consumed token.
pub fn last_span(&mut self) -> Span { pub fn recent_span(&mut self) -> Span {
self.token_buffer.first().map(|x| x.span).unwrap_or(self.last_span) self.token_buffer.first().map(|x| x.span).unwrap_or(self.last_span)
} }
/// returns the token of the last consumed token.
pub fn last_span(&mut self) -> Span {
self.last_span
}
/// Eat the next token if it is of the given kind. /// Eat the next token if it is of the given kind.
/// Returns whether a token was eaten. /// Returns whether a token was eaten.
pub fn eat(&mut self, token: TokenKind) -> bool { pub fn eat(&mut self, token: TokenKind) -> bool {
@ -187,7 +191,7 @@ impl<'a> Parser<'a> {
expected: kind, expected: kind,
should_close, should_close,
}, },
self.last_span(), self.recent_span(),
)); ));
} }
Ok(()) Ok(())
@ -214,24 +218,8 @@ impl<'a> Parser<'a> {
/// ///
/// This is the primary entry point of the parser. /// This is the primary entry point of the parser.
pub fn parse_query(&mut self) -> ParseResult<sql::Query> { pub fn parse_query(&mut self) -> ParseResult<sql::Query> {
// eat possible empty statements. let statements = self.parse_stmt_list()?;
while self.eat(t!(";")) {} Ok(sql::Query(statements))
let mut statements = vec![self.parse_stmt()?];
while self.eat(t!(";")) {
// eat possible empty statements.
while self.eat(t!(";")) {}
if let TokenKind::Eof = self.peek().kind {
break;
};
statements.push(self.parse_stmt()?);
}
expected!(self, TokenKind::Eof);
Ok(sql::Query(sql::Statements(statements)))
} }
/// Parse a single statement. /// Parse a single statement.

152
core/src/syn/v2/parser/prime.rs
View file

@ -3,8 +3,8 @@ use geo::Point;
use super::{ParseResult, Parser}; use super::{ParseResult, Parser};
use crate::{ use crate::{
sql::{ sql::{
Array, Dir, Function, Geometry, Ident, Idiom, Mock, Number, Part, Script, Strand, Subquery, Array, Dir, Function, Geometry, Ident, Idiom, Mock, Part, Script, Strand, Subquery, Table,
Table, Value, Value,
}, },
syn::v2::{ syn::v2::{
lexer::Lexer, lexer::Lexer,
@ -12,7 +12,7 @@ use crate::{
mac::{expected, unexpected}, mac::{expected, unexpected},
ParseError, ParseErrorKind, ParseError, ParseErrorKind,
}, },
token::{t, Span, TokenKind}, token::{t, NumberKind, Span, TokenKind},
}, },
}; };
@ -322,6 +322,7 @@ impl Parser<'_> {
match peek.kind { match peek.kind {
t!("(") => { t!("(") => {
self.pop_peek(); self.pop_peek();
dbg!("called");
self.parse_inner_subquery(Some(peek.span)) self.parse_inner_subquery(Some(peek.span))
} }
t!("IF") => { t!("IF") => {
@ -334,8 +335,8 @@ impl Parser<'_> {
} }
pub fn parse_inner_subquery_or_coordinate(&mut self, start: Span) -> ParseResult<Value> { pub fn parse_inner_subquery_or_coordinate(&mut self, start: Span) -> ParseResult<Value> {
let next = self.peek(); let peek = self.peek();
let res = match next.kind { let res = match peek.kind {
t!("RETURN") => { t!("RETURN") => {
self.pop_peek(); self.pop_peek();
let stmt = self.parse_return_stmt()?; let stmt = self.parse_return_stmt()?;
@ -376,40 +377,98 @@ impl Parser<'_> {
let stmt = self.parse_remove_stmt()?; let stmt = self.parse_remove_stmt()?;
Subquery::Remove(stmt) Subquery::Remove(stmt)
} }
t!("+") | t!("-") => {
// handle possible coordinate in the shape of ([-+]?number,[-+]?number)
if let TokenKind::Number(kind) = self.peek_token_at(1).kind {
// take the value so we don't overwrite it if the next token happens to be an
// strand or an ident, both of which are invalid syntax.
let number_value = self.lexer.string.take().unwrap();
if self.peek_token_at(2).kind == t!(",") {
match kind {
NumberKind::Decimal | NumberKind::NaN => {
return Err(ParseError::new(
ParseErrorKind::UnexpectedExplain {
found: TokenKind::Number(kind),
expected: "a non-decimal, non-nan number",
explain: "coordinate numbers can't be NaN or a decimal",
},
peek.span,
));
}
_ => {}
}
self.lexer.string = Some(number_value);
let a = self.parse_signed_float()?;
self.next();
let b = self.parse_signed_float()?;
self.expect_closing_delimiter(t!(")"), start)?;
return Ok(Value::Geometry(Geometry::Point(Point::from((a, b)))));
}
self.lexer.string = Some(number_value);
}
Subquery::Value(self.parse_value_field()?)
}
TokenKind::Number(kind) => {
// handle possible coordinate in the shape of ([-+]?number,[-+]?number)
// take the value so we don't overwrite it if the next token happens to be an
// strand or an ident, both of which are invalid syntax.
let number_value = self.lexer.string.take().unwrap();
if self.peek_token_at(1).kind == t!(",") {
match kind {
NumberKind::Decimal | NumberKind::NaN => {
return Err(ParseError::new(
ParseErrorKind::UnexpectedExplain {
found: TokenKind::Number(kind),
expected: "a non-decimal, non-nan number",
explain: "coordinate numbers can't be NaN or a decimal",
},
peek.span,
));
}
_ => {}
}
self.pop_peek();
// was a semicolon, put the strand back for code reuse.
self.lexer.string = Some(number_value);
let a = self.token_value::<f64>(peek)?;
// eat the semicolon.
self.next();
let b = self.parse_signed_float()?;
self.expect_closing_delimiter(t!(")"), start)?;
return Ok(Value::Geometry(Geometry::Point(Point::from((a, b)))));
}
self.lexer.string = Some(number_value);
Subquery::Value(self.parse_value_field()?)
}
_ => { _ => {
let value = self.parse_value_field()?; let value = self.parse_value_field()?;
Subquery::Value(value) Subquery::Value(value)
} }
}; };
match res { if self.peek_kind() != t!(")") && Self::starts_disallowed_subquery_statement(peek.kind) {
Subquery::Value(Value::Number(x)) => { if let Subquery::Value(Value::Idiom(Idiom(ref idiom))) = res {
if self.eat(t!(",")) { if idiom.len() == 1 {
// TODO: Fix number parsing. // we parsed a single idiom and the next token was a dissallowed statement so
let b = self.next_token_value::<Number>()?; // it is likely that the used meant to use an invalid statement.
return Err(ParseError::new(
let a: f64 = x ParseErrorKind::DisallowedStatement {
.try_into() found: self.peek_kind(),
.map_err(|_| ParseError::new(ParseErrorKind::Todo, next.span))?; expected: t!(")"),
let b: f64 = b disallowed: peek.span,
.try_into() },
.map_err(|_| ParseError::new(ParseErrorKind::Todo, next.span))?; self.recent_span(),
));
}
}
}
self.expect_closing_delimiter(t!(")"), start)?; self.expect_closing_delimiter(t!(")"), start)?;
Ok(Value::Geometry(Geometry::Point(Point::from((a, b))))) Ok(Value::Subquery(Box::new(res)))
} else {
self.expect_closing_delimiter(t!(")"), start)?;
Ok(Value::Subquery(Box::new(Subquery::Value(Value::Number(x)))))
}
}
x => {
self.expect_closing_delimiter(t!(")"), start)?;
Ok(Value::Subquery(Box::new(x)))
}
}
} }
pub fn parse_inner_subquery(&mut self, start: Option<Span>) -> ParseResult<Subquery> { pub fn parse_inner_subquery(&mut self, start: Option<Span>) -> ParseResult<Subquery> {
let res = match self.peek().kind { let peek = self.peek();
let res = match peek.kind {
t!("RETURN") => { t!("RETURN") => {
self.pop_peek(); self.pop_peek();
let stmt = self.parse_return_stmt()?; let stmt = self.parse_return_stmt()?;
@ -456,11 +515,44 @@ impl Parser<'_> {
} }
}; };
if let Some(start) = start { if let Some(start) = start {
if self.peek_kind() != t!(")") && Self::starts_disallowed_subquery_statement(peek.kind)
{
if let Subquery::Value(Value::Idiom(Idiom(ref idiom))) = res {
if idiom.len() == 1 {
// we parsed a single idiom and the next token was a dissallowed statement so
// it is likely that the used meant to use an invalid statement.
return Err(ParseError::new(
ParseErrorKind::DisallowedStatement {
found: self.peek_kind(),
expected: t!(")"),
disallowed: peek.span,
},
self.recent_span(),
));
}
}
}
self.expect_closing_delimiter(t!(")"), start)?; self.expect_closing_delimiter(t!(")"), start)?;
} }
Ok(res) Ok(res)
} }
fn starts_disallowed_subquery_statement(kind: TokenKind) -> bool {
matches!(
kind,
t!("ANALYZE")
| t!("BEGIN") | t!("BREAK")
| t!("CANCEL") | t!("COMMIT")
| t!("CONTINUE") | t!("FOR")
| t!("INFO") | t!("KILL")
| t!("LIVE") | t!("OPTION")
| t!("LET") | t!("SHOW")
| t!("SLEEP") | t!("THROW")
| t!("USE")
)
}
/// Parses a strand with legacy rules, parsing to a record id, datetime or uuid if the string /// Parses a strand with legacy rules, parsing to a record id, datetime or uuid if the string
/// matches. /// matches.
pub fn parse_legacy_strand(&mut self) -> ParseResult<Value> { pub fn parse_legacy_strand(&mut self) -> ParseResult<Value> {

39
core/src/syn/v2/parser/stmt/mod.rs
View file

@ -5,6 +5,7 @@ use crate::sql::statements::{
KillStatement, LiveStatement, OptionStatement, SetStatement, ThrowStatement, KillStatement, LiveStatement, OptionStatement, SetStatement, ThrowStatement,
}; };
use crate::sql::{Fields, Ident, Param}; use crate::sql::{Fields, Ident, Param};
use crate::syn::v2::parser::{ParseError, ParseErrorKind};
use crate::syn::v2::token::{t, TokenKind}; use crate::syn::v2::token::{t, TokenKind};
use crate::{ use crate::{
sql::{ sql::{
@ -43,15 +44,51 @@ impl Parser<'_> {
let stmt = self.parse_stmt()?; let stmt = self.parse_stmt()?;
res.push(stmt); res.push(stmt);
if !self.eat(t!(";")) { if !self.eat(t!(";")) {
expected!(self, t!("eof")); if self.eat(t!("eof")) {
break; break;
} }
if Self::token_kind_starts_statement(self.peek_kind()) {
// user likely forgot a semicolon.
return Err(ParseError::new(
ParseErrorKind::UnexpectedExplain {
found: self.peek_kind(),
expected: "the query to end",
explain:
"maybe forgot a semicolon after the previous statement?",
},
self.recent_span(),
));
}
expected!(self, t!("eof"));
}
} }
} }
} }
Ok(Statements(res)) Ok(Statements(res))
} }
fn token_kind_starts_statement(kind: TokenKind) -> bool {
matches!(
kind,
t!("ANALYZE")
| t!("BEGIN") | t!("BREAK")
| t!("CANCEL") | t!("COMMIT")
| t!("CONTINUE") | t!("CREATE")
| t!("DEFINE") | t!("DELETE")
| t!("FOR") | t!("IF")
| t!("INFO") | t!("INSERT")
| t!("KILL") | t!("LIVE")
| t!("OPTION") | t!("RETURN")
| t!("RELATE") | t!("REMOVE")
| t!("SELECT") | t!("LET")
| t!("SHOW") | t!("SLEEP")
| t!("THROW") | t!("UPDATE")
| t!("USE")
)
}
pub(super) fn parse_stmt(&mut self) -> ParseResult<Statement> { pub(super) fn parse_stmt(&mut self) -> ParseResult<Statement> {
let token = self.peek(); let token = self.peek();
match token.kind { match token.kind {

126
core/src/syn/v2/parser/stmt/parts.rs
View file

@ -2,15 +2,17 @@
use crate::{ use crate::{
sql::{ sql::{
changefeed::ChangeFeed, index::Distance, Base, Cond, Data, Duration, Fetch, Fetchs, Group, changefeed::ChangeFeed, index::Distance, Base, Cond, Data, Duration, Fetch, Fetchs, Field,
Groups, Ident, Operator, Output, Permission, Permissions, Tables, Timeout, View, Fields, Group, Groups, Ident, Idiom, Output, Permission, Permissions, Tables, Timeout,
Value, View,
}, },
syn::v2::{ syn::v2::{
parser::{ parser::{
error::MissingKind,
mac::{expected, unexpected}, mac::{expected, unexpected},
ParseResult, Parser, ParseError, ParseErrorKind, ParseResult, Parser,
}, },
token::{t, DistanceKind, TokenKind}, token::{t, DistanceKind, Span, TokenKind},
}, },
}; };
@ -24,13 +26,7 @@ impl Parser<'_> {
let mut set_list = Vec::new(); let mut set_list = Vec::new();
loop { loop {
let idiom = self.parse_plain_idiom()?; let idiom = self.parse_plain_idiom()?;
let operator = match self.next().kind { let operator = self.parse_assigner()?;
t!("=") => Operator::Equal,
t!("+=") => Operator::Inc,
t!("-=") => Operator::Dec,
t!("+?=") => Operator::Ext,
x => unexpected!(self, x, "a assign operator"),
};
let value = self.parse_value()?; let value = self.parse_value()?;
set_list.push((idiom, operator, value)); set_list.push((idiom, operator, value));
if !self.eat(t!(",")) { if !self.eat(t!(",")) {
@ -121,34 +117,94 @@ impl Parser<'_> {
Ok(Some(Cond(v))) Ok(Some(Cond(v)))
} }
pub fn try_parse_group(&mut self) -> ParseResult<Option<Groups>> { pub fn check_idiom<'a>(
kind: MissingKind,
fields: &'a Fields,
field_span: Span,
idiom: &Idiom,
idiom_span: Span,
) -> ParseResult<&'a Field> {
let mut found = None;
for field in fields.iter() {
let Field::Single {
expr,
alias,
} = field
else {
unreachable!()
};
if let Some(alias) = alias {
if idiom == alias {
found = Some(field);
break;
}
}
match expr {
Value::Idiom(x) => {
if idiom == x {
found = Some(field);
break;
}
}
v => {
if *idiom == v.to_idiom() {
found = Some(field);
break;
}
}
}
}
found.ok_or_else(|| {
ParseError::new(
ParseErrorKind::MissingField {
field: field_span,
idiom: idiom.to_string(),
kind,
},
idiom_span,
)
})
}
pub fn try_parse_group(
&mut self,
fields: &Fields,
fields_span: Span,
) -> ParseResult<Option<Groups>> {
if !self.eat(t!("GROUP")) { if !self.eat(t!("GROUP")) {
return Ok(None); return Ok(None);
} }
let res = match self.peek_kind() { if self.eat(t!("ALL")) {
t!("ALL") => { return Ok(Some(Groups(Vec::new())));
self.pop_peek();
Groups(Vec::new())
} }
t!("BY") => {
self.pop_peek();
let mut groups = Groups(vec![Group(self.parse_basic_idiom()?)]);
while self.eat(t!(",")) {
groups.0.push(Group(self.parse_basic_idiom()?));
}
groups
}
_ => {
let mut groups = Groups(vec![Group(self.parse_basic_idiom()?)]);
while self.eat(t!(",")) {
groups.0.push(Group(self.parse_basic_idiom()?));
}
groups
}
};
Ok(Some(res)) self.eat(t!("BY"));
let has_all = fields.contains(&Field::All);
let before = self.peek().span;
let group = self.parse_basic_idiom()?;
let group_span = before.covers(self.last_span());
if !has_all {
Self::check_idiom(MissingKind::Group, fields, fields_span, &group, group_span)?;
}
let mut groups = Groups(vec![Group(group)]);
while self.eat(t!(",")) {
let before = self.peek().span;
let group = self.parse_basic_idiom()?;
let group_span = before.covers(self.last_span());
if !has_all {
Self::check_idiom(MissingKind::Group, fields, fields_span, &group, group_span)?;
}
groups.0.push(Group(group));
}
Ok(Some(groups))
} }
/// Parse a permissions production /// Parse a permissions production
@ -287,7 +343,9 @@ impl Parser<'_> {
/// parens. Expects the next keyword to be `SELECT`. /// parens. Expects the next keyword to be `SELECT`.
pub fn parse_view(&mut self) -> ParseResult<View> { pub fn parse_view(&mut self) -> ParseResult<View> {
expected!(self, t!("SELECT")); expected!(self, t!("SELECT"));
let before_fields = self.peek().span;
let fields = self.parse_fields()?; let fields = self.parse_fields()?;
let fields_span = before_fields.covers(self.recent_span());
expected!(self, t!("FROM")); expected!(self, t!("FROM"));
let mut from = vec![self.next_token_value()?]; let mut from = vec![self.next_token_value()?];
while self.eat(t!(",")) { while self.eat(t!(",")) {
@ -295,7 +353,7 @@ impl Parser<'_> {
} }
let cond = self.try_parse_condition()?; let cond = self.try_parse_condition()?;
let group = self.try_parse_group()?; let group = self.try_parse_group(&fields, fields_span)?;
Ok(View { Ok(View {
expr: fields, expr: fields,

82
core/src/syn/v2/parser/stmt/select.rs
View file

@ -1,21 +1,23 @@
use crate::{ use crate::{
sql::{ sql::{
statements::SelectStatement, Explain, Ident, Idioms, Limit, Order, Orders, Split, Splits, statements::SelectStatement, Explain, Field, Fields, Ident, Idioms, Limit, Order, Orders,
Start, Values, Version, With, Split, Splits, Start, Values, Version, With,
}, },
syn::v2::{ syn::v2::{
parser::{ parser::{
error::MissingKind,
mac::{expected, unexpected}, mac::{expected, unexpected},
ParseResult, Parser, ParseResult, Parser,
}, },
token::t, token::{t, Span},
}, },
}; };
impl Parser<'_> { impl Parser<'_> {
pub(crate) fn parse_select_stmt(&mut self) -> ParseResult<SelectStatement> { pub(crate) fn parse_select_stmt(&mut self) -> ParseResult<SelectStatement> {
// let before = self.peek().span;
let expr = self.parse_fields()?; let expr = self.parse_fields()?;
let fields_span = before.covers(self.last_span());
let omit = self.eat(t!("OMIT")).then(|| self.parse_idiom_list()).transpose()?.map(Idioms); let omit = self.eat(t!("OMIT")).then(|| self.parse_idiom_list()).transpose()?.map(Idioms);
@ -31,9 +33,9 @@ impl Parser<'_> {
let with = self.try_parse_with()?; let with = self.try_parse_with()?;
let cond = self.try_parse_condition()?; let cond = self.try_parse_condition()?;
let split = self.try_parse_split()?; let split = self.try_parse_split(&expr, fields_span)?;
let group = self.try_parse_group()?; let group = self.try_parse_group(&expr, fields_span)?;
let order = self.try_parse_orders()?; let order = self.try_parse_orders(&expr, fields_span)?;
let (limit, start) = if let t!("START") = self.peek_kind() { let (limit, start) = if let t!("START") = self.peek_kind() {
let start = self.try_parse_start()?; let start = self.try_parse_start()?;
let limit = self.try_parse_limit()?; let limit = self.try_parse_limit()?;
@ -91,49 +93,83 @@ impl Parser<'_> {
Ok(Some(with)) Ok(Some(with))
} }
fn try_parse_split(&mut self) -> ParseResult<Option<Splits>> { fn try_parse_split(
&mut self,
fields: &Fields,
fields_span: Span,
) -> ParseResult<Option<Splits>> {
if !self.eat(t!("SPLIT")) { if !self.eat(t!("SPLIT")) {
return Ok(None); return Ok(None);
} }
self.eat(t!("ON")); self.eat(t!("ON"));
let mut res = vec![Split(self.parse_basic_idiom()?)]; let has_all = fields.contains(&Field::All);
let before = self.peek().span;
let split = self.parse_basic_idiom()?;
let split_span = before.covers(self.last_span());
if !has_all {
Self::check_idiom(MissingKind::Split, fields, fields_span, &split, split_span)?;
}
let mut res = vec![Split(split)];
while self.eat(t!(",")) { while self.eat(t!(",")) {
res.push(Split(self.parse_basic_idiom()?)); let before = self.peek().span;
let split = self.parse_basic_idiom()?;
let split_span = before.covers(self.last_span());
if !has_all {
Self::check_idiom(MissingKind::Split, fields, fields_span, &split, split_span)?;
}
res.push(Split(split))
} }
Ok(Some(Splits(res))) Ok(Some(Splits(res)))
} }
fn try_parse_orders(&mut self) -> ParseResult<Option<Orders>> { fn try_parse_orders(
&mut self,
fields: &Fields,
fields_span: Span,
) -> ParseResult<Option<Orders>> {
if !self.eat(t!("ORDER")) { if !self.eat(t!("ORDER")) {
return Ok(None); return Ok(None);
} }
self.eat(t!("BY")); self.eat(t!("BY"));
let orders = match self.peek_kind() { if let t!("RAND") = self.peek_kind() {
t!("RAND") => {
self.pop_peek(); self.pop_peek();
let start = expected!(self, t!("(")).span; let start = expected!(self, t!("(")).span;
self.expect_closing_delimiter(t!(")"), start)?; self.expect_closing_delimiter(t!(")"), start)?;
vec![Order { return Ok(Some(Orders(vec![Order {
order: Default::default(), order: Default::default(),
random: true, random: true,
collate: false, collate: false,
numeric: false, numeric: false,
direction: true, direction: true,
}] }])));
}
_ => {
let mut orders = vec![self.parse_order()?];
while self.eat(t!(",")) {
orders.push(self.parse_order()?);
}
orders
}
}; };
let has_all = fields.contains(&Field::All);
let before = self.recent_span();
let order = self.parse_order()?;
let order_span = before.covers(self.last_span());
if !has_all {
Self::check_idiom(MissingKind::Order, fields, fields_span, &order, order_span)?;
}
let mut orders = vec![order];
while self.eat(t!(",")) {
let before = self.recent_span();
let order = self.parse_order()?;
let order_span = before.covers(self.last_span());
if !has_all {
Self::check_idiom(MissingKind::Order, fields, fields_span, &order, order_span)?;
}
orders.push(order)
}
Ok(Some(Orders(orders))) Ok(Some(Orders(orders)))
} }

23
core/src/syn/v2/parser/thing.rs
View file

@ -93,8 +93,27 @@ impl Parser<'_> {
end, end,
}))) })))
} else { } else {
let Bound::Included(id) = beg else { let id = match beg {
unexpected!(self, self.peek_kind(), "the range operator '..'") Bound::Unbounded => {
if self.peek_kind() == t!("$param") {
return Err(ParseError::new(
ParseErrorKind::UnexpectedExplain {
found: t!("$param"),
expected: "a record-id id",
explain: "you can create a record-id from a param with the function 'type::thing'",
},
self.recent_span(),
));
}
// we haven't matched anythong so far so we still want any type of id.
unexpected!(self, self.peek_kind(), "a record-id id")
}
Bound::Excluded(_) => {
// we have matched a bounded id but we don't see an range operator.
unexpected!(self, self.peek_kind(), "the range operator `..`")
}
Bound::Included(id) => id,
}; };
Ok(Value::Thing(Thing { Ok(Value::Thing(Thing {
tb: ident, tb: ident,

138
core/src/syn/v2/token/mod.rs
View file

@ -42,6 +42,14 @@ impl Span {
len, len,
} }
} }
// returns a zero-length span that starts after the current span.
pub fn after(self) -> Span {
Span {
offset: self.offset + self.len,
len: 0,
}
}
} }
#[repr(u8)] #[repr(u8)]
@ -124,42 +132,42 @@ pub enum Operator {
impl Operator { impl Operator {
fn as_str(&self) -> &'static str { fn as_str(&self) -> &'static str {
match self { match self {
Operator::Not => "'!'", Operator::Not => "!",
Operator::Add => "'+'", Operator::Add => "+",
Operator::Subtract => "'-'", Operator::Subtract => "-",
Operator::Divide => "'÷'", Operator::Divide => "÷",
Operator::Or => "'||'", Operator::Or => "||",
Operator::And => "'&&'", Operator::And => "&&",
Operator::Mult => "'×'", Operator::Mult => "×",
Operator::LessEqual => "'<='", Operator::LessEqual => "<=",
Operator::GreaterEqual => "'>='", Operator::GreaterEqual => ">=",
Operator::Star => "'*'", Operator::Star => "*",
Operator::Power => "'**'", Operator::Power => "**",
Operator::Equal => "'='", Operator::Equal => "=",
Operator::Exact => "'=='", Operator::Exact => "==",
Operator::NotEqual => "'!='", Operator::NotEqual => "!=",
Operator::AllEqual => "'*='", Operator::AllEqual => "*=",
Operator::AnyEqual => "'?='", Operator::AnyEqual => "?=",
Operator::Like => "'~'", Operator::Like => "~",
Operator::NotLike => "'!~'", Operator::NotLike => "!~",
Operator::AllLike => "'*~'", Operator::AllLike => "*~",
Operator::AnyLike => "'?~'", Operator::AnyLike => "?~",
Operator::Contains => "'∋'", Operator::Contains => "∋",
Operator::NotContains => "'∌'", Operator::NotContains => "∌",
Operator::ContainsAll => "'⊇'", Operator::ContainsAll => "⊇",
Operator::ContainsAny => "'⊃'", Operator::ContainsAny => "⊃",
Operator::ContainsNone => "'⊅'", Operator::ContainsNone => "⊅",
Operator::Inside => "'∈'", Operator::Inside => "∈",
Operator::NotInside => "'∉'", Operator::NotInside => "∉",
Operator::AllInside => "'⊆'", Operator::AllInside => "⊆",
Operator::AnyInside => "'⊂'", Operator::AnyInside => "⊂",
Operator::NoneInside => "'⊄'", Operator::NoneInside => "⊄",
Operator::Matches => "'@@'", Operator::Matches => "@@",
Operator::Inc => "'+='", Operator::Inc => "+=",
Operator::Dec => "'-='", Operator::Dec => "-=",
Operator::Ext => "'+?='", Operator::Ext => "+?=",
Operator::Tco => "'?:'", Operator::Tco => "?:",
Operator::Nco => "'??'", Operator::Nco => "??",
} }
} }
} }
@ -308,11 +316,31 @@ impl TokenKind {
) )
} }
fn algorithm_as_str(algo: Algorithm) -> &'static str {
match algo {
Algorithm::EdDSA => "EDDSA",
Algorithm::Es256 => "ES256",
Algorithm::Es384 => "ES384",
Algorithm::Es512 => "ES512",
Algorithm::Hs256 => "HS256",
Algorithm::Hs384 => "HS384",
Algorithm::Hs512 => "HS512",
Algorithm::Ps256 => "PS256",
Algorithm::Ps384 => "PS384",
Algorithm::Ps512 => "PS512",
Algorithm::Rs256 => "RS256",
Algorithm::Rs384 => "RS384",
Algorithm::Rs512 => "RS512",
#[cfg(feature = "sql2")]
Algorithm::Jwks => "JWKS",
}
}
pub fn as_str(&self) -> &'static str { pub fn as_str(&self) -> &'static str {
match *self { match *self {
TokenKind::Keyword(x) => x.as_str(), TokenKind::Keyword(x) => x.as_str(),
TokenKind::Operator(x) => x.as_str(), TokenKind::Operator(x) => x.as_str(),
TokenKind::Algorithm(_) => todo!(), TokenKind::Algorithm(x) => Self::algorithm_as_str(x),
TokenKind::Language(x) => x.as_str(), TokenKind::Language(x) => x.as_str(),
TokenKind::Distance(x) => x.as_str(), TokenKind::Distance(x) => x.as_str(),
TokenKind::OpenDelim(Delim::Paren) => "(", TokenKind::OpenDelim(Delim::Paren) => "(",
@ -335,24 +363,24 @@ impl TokenKind {
TokenKind::Number(_) => "a number", TokenKind::Number(_) => "a number",
TokenKind::Identifier => "an identifier", TokenKind::Identifier => "an identifier",
TokenKind::Regex => "a regex", TokenKind::Regex => "a regex",
TokenKind::LeftChefron => "'<'", TokenKind::LeftChefron => "<",
TokenKind::RightChefron => "'>'", TokenKind::RightChefron => ">",
TokenKind::Star => "'*'", TokenKind::Star => "*",
TokenKind::Dollar => "'$'", TokenKind::Dollar => "$",
TokenKind::Question => "'?'", TokenKind::Question => "?",
TokenKind::ArrowRight => "'->'", TokenKind::ArrowRight => "->",
TokenKind::ArrowLeft => "'<-'", TokenKind::ArrowLeft => "<-",
TokenKind::BiArrow => "'<->'", TokenKind::BiArrow => "<->",
TokenKind::ForwardSlash => "'/'", TokenKind::ForwardSlash => "/",
TokenKind::Dot => "'.'", TokenKind::Dot => ".",
TokenKind::DotDot => "'..'", TokenKind::DotDot => "..",
TokenKind::DotDotDot => "'...'", TokenKind::DotDotDot => "...",
TokenKind::SemiColon => "';'", TokenKind::SemiColon => ";",
TokenKind::PathSeperator => "'::'", TokenKind::PathSeperator => "::",
TokenKind::Colon => "':'", TokenKind::Colon => ":",
TokenKind::Comma => "','", TokenKind::Comma => ",",
TokenKind::Vert => "'|'", TokenKind::Vert => "|",
TokenKind::At => "'@'", TokenKind::At => "@",
TokenKind::Invalid => "Invalid", TokenKind::Invalid => "Invalid",
TokenKind::Eof => "Eof", TokenKind::Eof => "Eof",
} }