From 4af24a5ca08cf8627ca5dbdc7b06ac6ca2cad2b9 Mon Sep 17 00:00:00 2001 From: Tobie Morgan Hitchcock Date: Thu, 21 Jul 2016 22:48:32 +0100 Subject: [PATCH] Add functionality to database defined fields --- db/define.go | 3 + sql/ast.go | 16 ++- sql/exprs.go | 14 ++ sql/field.go | 19 ++- sql/type.go | 8 +- util/conv/conv.go | 155 ++++++++++++++++++++++ util/item/item.go | 319 ++++++++++++++++++++++++++++++++++++++++++++-- 7 files changed, 512 insertions(+), 22 deletions(-) create mode 100644 util/conv/conv.go diff --git a/db/define.go b/db/define.go index bcaf9f71..4c874e7d 100644 --- a/db/define.go +++ b/db/define.go @@ -63,13 +63,16 @@ func executeDefineFieldStatement(ast *sql.DefineFieldStatement) (out []interface doc := data.New() doc.Set(ast.Name, "name") doc.Set(ast.Type, "type") + doc.Set(ast.Enum, "enum") doc.Set(ast.Code, "code") doc.Set(ast.Min, "min") doc.Set(ast.Max, "max") + doc.Set(ast.Match, "match") doc.Set(ast.Default, "default") doc.Set(ast.Notnull, "notnull") doc.Set(ast.Readonly, "readonly") doc.Set(ast.Mandatory, "mandatory") + doc.Set(ast.Validate, "validate") for _, TB := range ast.What { diff --git a/sql/ast.go b/sql/ast.go index e98273fe..1277b787 100644 --- a/sql/ast.go +++ b/sql/ast.go @@ -182,8 +182,8 @@ type RemoveTableStatement struct { // DefineFieldStatement represents an SQL DEFINE INDEX statement. // // DEFINE FIELD name ON person TYPE string CODE {} -// DEFINE FIELD name ON person TYPE [0,1,2,3,4,5] DEFAULT 0 -// DEFINE FIELD name ON person TYPE [0...100]number MIN 0 MAX 3 DEFAULT 0 +// DEFINE FIELD name ON person TYPE number MIN 0 MAX 5 DEFAULT 0 +// DEFINE FIELD name ON person TYPE custom ENUM [0,1,2,3,4,5] DEFAULT 0 type DefineFieldStatement struct { EX bool // Explain KV string // Bucket @@ -191,15 +191,17 @@ type DefineFieldStatement struct { DB string // Database Name Ident // Field name What []Table // Table names - Type Expr // Field type + Type Ident // Field type Enum []interface{} // Custom options Code string // Field code Min float64 // Minimum value / length Max float64 // Maximum value / length - Default Expr // Default value - Notnull bool // Notnull? - Readonly bool // Readonly? - Mandatory bool // Mnadatory? + Match string // Regex value + Default interface{} // Default value + Notnull bool // Notnull - can not be NULL? + Readonly bool // Readonly - can not be changed? + Mandatory bool // Mandatory - can not be VOID? + Validate bool // Validate - can not be INCORRECT? } // RemoveFieldStatement represents an SQL REMOVE INDEX statement. diff --git a/sql/exprs.go b/sql/exprs.go index 099fd5c1..499f952e 100644 --- a/sql/exprs.go +++ b/sql/exprs.go @@ -15,6 +15,7 @@ package sql import ( + "regexp" "strconv" "time" ) @@ -228,6 +229,19 @@ func (p *Parser) parseScript() (string, error) { } +func (p *Parser) parseRegexp() (string, error) { + + tok, lit, err := p.shouldBe(REGEX) + if err != nil { + return string(""), &ParseError{Found: lit, Expected: []string{"regular expression"}} + } + + val, err := declare(tok, lit) + + return val.(*regexp.Regexp).String(), err + +} + func (p *Parser) parseBoolean() (bool, error) { tok, lit, err := p.shouldBe(TRUE, FALSE) diff --git a/sql/field.go b/sql/field.go index 9d3d00d4..c69e1d91 100644 --- a/sql/field.go +++ b/sql/field.go @@ -38,7 +38,7 @@ func (p *Parser) parseDefineFieldStatement(explain bool) (stmt *DefineFieldState for { - tok, _, exi := p.mightBe(MIN, MAX, TYPE, ENUM, CODE, DEFAULT, NOTNULL, READONLY, MANDATORY) + tok, _, exi := p.mightBe(MIN, MAX, TYPE, ENUM, CODE, MATCH, DEFAULT, NOTNULL, READONLY, MANDATORY, VALIDATE) if !exi { break } @@ -73,6 +73,12 @@ func (p *Parser) parseDefineFieldStatement(explain bool) (stmt *DefineFieldState } } + if is(tok, MATCH) { + if stmt.Match, err = p.parseRegexp(); err != nil { + return nil, err + } + } + if is(tok, DEFAULT) { if stmt.Default, err = p.parseDefault(); err != nil { return nil, err @@ -106,9 +112,18 @@ func (p *Parser) parseDefineFieldStatement(explain bool) (stmt *DefineFieldState } } + if is(tok, VALIDATE) { + stmt.Validate = true + if tok, _, exi := p.mightBe(TRUE, FALSE); exi { + if tok == FALSE { + stmt.Validate = false + } + } + } + } - if stmt.Type == nil { + if stmt.Type == "" { return nil, &ParseError{Found: "", Expected: []string{"TYPE"}} } diff --git a/sql/type.go b/sql/type.go index c401e90d..18c577fa 100644 --- a/sql/type.go +++ b/sql/type.go @@ -14,17 +14,17 @@ package sql -func (p *Parser) parseType() (exp Expr, err error) { +func (p *Parser) parseType() (exp Ident, err error) { - allowed := []string{"any", "url", "email", "phone", "array", "object", "string", "number", "custom", "boolean", "datetime"} + allowed := []string{"any", "url", "uuid", "color", "email", "phone", "array", "object", "domain", "string", "number", "custom", "boolean", "datetime", "latitude", "longitude"} tok, lit, err := p.shouldBe(IDENT) if err != nil { - return nil, err + return Ident(""), err } if !contains(lit, allowed) { - return nil, &ParseError{Found: lit, Expected: allowed} + return Ident(""), &ParseError{Found: lit, Expected: allowed} } val, err := declare(tok, lit) diff --git a/util/conv/conv.go b/util/conv/conv.go new file mode 100644 index 00000000..ba82eaea --- /dev/null +++ b/util/conv/conv.go @@ -0,0 +1,155 @@ +// Copyright © 2016 Abcum Ltd +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package conv + +import ( + "fmt" + "time" + + "github.com/asaskevich/govalidator" +) + +func ConvertToUrl(obj interface{}) (val string, err error) { + val = govalidator.ToString(obj) + if !govalidator.IsURL(val) { + err = fmt.Errorf("Not a valid url") + } + return +} + +func ConvertToUuid(obj interface{}) (val string, err error) { + val = govalidator.ToString(obj) + if !govalidator.IsUUID(val) { + err = fmt.Errorf("Not a valid uuid") + } + return +} + +func ConvertToEmail(obj interface{}) (val string, err error) { + val = govalidator.ToString(obj) + if !govalidator.IsEmail(val) { + err = fmt.Errorf("Not a valid email") + } + return govalidator.NormalizeEmail(val) +} + +func ConvertToPhone(obj interface{}) (val string, err error) { + val = govalidator.ToString(obj) + if !govalidator.Matches(val, `^[\s\d\+\-\(\)]+$`) { + err = fmt.Errorf("Not a valid phone") + } + return +} + +func ConvertToColor(obj interface{}) (val string, err error) { + val = govalidator.ToString(obj) + if !govalidator.IsHexcolor(val) && !govalidator.IsRGBcolor(val) { + err = fmt.Errorf("Not a valid color") + } + return +} + +func ConvertToArray(obj interface{}) (val []interface{}, err error) { + if now, ok := obj.([]interface{}); ok { + val = now + } else { + err = fmt.Errorf("Not a valid array") + } + return +} + +func ConvertToObject(obj interface{}) (val map[string]interface{}, err error) { + if now, ok := obj.(map[string]interface{}); ok { + val = now + } else { + err = fmt.Errorf("Not a valid object") + } + return +} + +func ConvertToDomain(obj interface{}) (val string, err error) { + val = govalidator.ToString(obj) + if !govalidator.IsDNSName(val) { + err = fmt.Errorf("Not a valid domain name") + } + return +} + +func ConvertToBase64(obj interface{}) (val string, err error) { + val = govalidator.ToString(obj) + if !govalidator.IsBase64(val) { + err = fmt.Errorf("Not valid base64 data") + } + return +} + +func ConvertToString(obj interface{}) (val string, err error) { + if now, ok := obj.(string); ok { + return now, err + } + return govalidator.ToString(obj), err +} + +func ConvertToNumber(obj interface{}) (val float64, err error) { + if now, ok := obj.(float64); ok { + return now, err + } + return govalidator.ToFloat(govalidator.ToString(obj)) +} + +func ConvertToBoolean(obj interface{}) (val bool, err error) { + if now, ok := obj.(bool); ok { + return now, err + } + return govalidator.ToBoolean(govalidator.ToString(obj)) +} + +func ConvertToDatetime(obj interface{}) (val time.Time, err error) { + if now, ok := obj.(time.Time); ok { + val = now + } else { + err = fmt.Errorf("Not a valid datetime") + } + return +} + +func ConvertToLatitude(obj interface{}) (val float64, err error) { + str := govalidator.ToString(obj) + if !govalidator.IsLatitude(str) { + err = fmt.Errorf("Not a valid latitude") + } + return govalidator.ToFloat(str) +} + +func ConvertToLongitude(obj interface{}) (val float64, err error) { + str := govalidator.ToString(obj) + if !govalidator.IsLatitude(str) { + err = fmt.Errorf("Not a valid longitude") + } + return govalidator.ToFloat(str) +} + +func ConvertToOneOf(obj interface{}, pos ...interface{}) (val interface{}, err error) { + for _, now := range pos { + if num, ok := obj.(int64); ok { + if float64(num) == now { + return obj, nil + } + } else if obj == now { + return obj, nil + } + } + return nil, fmt.Errorf("Not a valid option") +} diff --git a/util/item/item.go b/util/item/item.go index ebd216a3..93bd3060 100644 --- a/util/item/item.go +++ b/util/item/item.go @@ -16,6 +16,7 @@ package item import ( "fmt" + "regexp" "time" "github.com/imdario/mergo" @@ -23,6 +24,7 @@ import ( "github.com/abcum/surreal/kvs" "github.com/abcum/surreal/sql" + "github.com/abcum/surreal/util/conv" "github.com/abcum/surreal/util/data" // "github.com/abcum/surreal/util/diff" "github.com/abcum/surreal/util/keys" @@ -33,12 +35,14 @@ type field struct { Name string Code string Enum []interface{} - Min int64 - Max int64 + Min float64 + Max float64 + Match string Default interface{} Notnull bool Readonly bool Mandatory bool + Validate bool } type index struct { @@ -329,15 +333,18 @@ func (this *Doc) getFlds(txn kvs.TX) (out []*field) { fld := &field{} - fld.Type, _ = inf.Get("type").Data().(string) fld.Name, _ = inf.Get("name").Data().(string) + fld.Type, _ = inf.Get("type").Data().(string) + fld.Enum, _ = inf.Get("enum").Data().([]interface{}) fld.Code, _ = inf.Get("code").Data().(string) - fld.Min, _ = inf.Get("min").Data().(int64) - fld.Max, _ = inf.Get("max").Data().(int64) + fld.Min, _ = inf.Get("min").Data().(float64) + fld.Max, _ = inf.Get("max").Data().(float64) + fld.Match, _ = inf.Get("match").Data().(string) fld.Default = inf.Get("default").Data() fld.Notnull = inf.Get("notnull").Data().(bool) fld.Readonly = inf.Get("readonly").Data().(bool) fld.Mandatory = inf.Get("mandatory").Data().(bool) + fld.Validate = inf.Get("validate").Data().(bool) out = append(out, fld) @@ -379,7 +386,11 @@ func (this *Doc) mrgFld(txn kvs.TX) (err error) { for _, fld := range this.getFlds(txn) { - initial := this.initial.Get("data", fld.Name).Data() + var exists bool + var initial interface{} + var current interface{} + + initial = this.initial.Get("data", fld.Name).Data() if fld.Readonly && initial != nil { this.current.Set(initial, "data", fld.Name) @@ -408,8 +419,8 @@ func (this *Doc) mrgFld(txn kvs.TX) (err error) { } - current := this.current.Get("data", fld.Name).Data() - exists := this.current.Exists("data", fld.Name) + current = this.current.Get("data", fld.Name).Data() + exists = this.current.Exists("data", fld.Name) if fld.Default != nil && exists == false { this.current.Set(fld.Default, "data", fld.Name) @@ -423,8 +434,298 @@ func (this *Doc) mrgFld(txn kvs.TX) (err error) { return fmt.Errorf("Need to set field '%v'", fld.Name) } - if fld.Type != "" { + if current != nil && fld.Type != "" { + switch fld.Type { + + case "url": + if val, err := conv.ConvertToUrl(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be a URL", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "uuid": + if val, err := conv.ConvertToUuid(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be a UUID", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "color": + if val, err := conv.ConvertToColor(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be a HEX or RGB color", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "email": + if val, err := conv.ConvertToEmail(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be a email address", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "phone": + if val, err := conv.ConvertToPhone(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be a phone number", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "array": + if val, err := conv.ConvertToArray(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be a array", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "object": + if val, err := conv.ConvertToObject(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be a object", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "domain": + if val, err := conv.ConvertToDomain(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be a domain name", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "base64": + if val, err := conv.ConvertToBase64(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be base64 data", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "string": + if val, err := conv.ConvertToString(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be a string", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "number": + if val, err := conv.ConvertToNumber(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be a number", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "boolean": + if val, err := conv.ConvertToBoolean(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be a boolean", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "datetime": + if val, err := conv.ConvertToDatetime(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be a datetime", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "latitude": + if val, err := conv.ConvertToLatitude(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be a latitude value", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "longitude": + if val, err := conv.ConvertToLongitude(current); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be a longitude value", fld.Name) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case "custom": + + if val, err := conv.ConvertToOneOf(current, fld.Enum...); err == nil { + this.current.Set(val, "data", fld.Name) + } else { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be one of %v", fld.Name, fld.Enum) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + } + + } + + if fld.Match != "" { + + if reg, err := regexp.Compile(fld.Match); err != nil { + return fmt.Errorf("Regular expression /%v/ is invalid", fld.Match) + } else { + if !reg.MatchString(fmt.Sprintf("%v", current)) { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to match the regular expression /%v/", fld.Name, fld.Match) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + } + + } + + if fld.Min != 0 { + + if current = this.current.Get("data", fld.Name).Data(); current != nil { + + switch now := current.(type) { + + case []interface{}: + if len(now) < int(fld.Min) { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be have at least %v items", fld.Name, fld.Min) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case string: + if len(now) < int(fld.Min) { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to at least %v characters", fld.Name, fld.Min) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case float64: + if now < fld.Min { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be >= %v", fld.Name, fld.Min) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + } + + } + + } + + if fld.Max != 0 { + + if current = this.current.Get("data", fld.Name).Data(); current != nil { + + switch now := current.(type) { + + case []interface{}: + if len(now) > int(fld.Max) { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be have %v or fewer items", fld.Name, fld.Max) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case string: + if len(now) > int(fld.Max) { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be %v or fewer characters", fld.Name, fld.Max) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + case float64: + if now > fld.Max { + if fld.Validate { + return fmt.Errorf("Field '%v' needs to be <= %v", fld.Name, fld.Max) + } else { + this.current.Try(initial, "data", fld.Name) + } + } + + } + + } + + } + + current = this.current.Get("data", fld.Name).Data() + exists = this.current.Exists("data", fld.Name) + + if fld.Default != nil && exists == false { + this.current.Set(fld.Default, "data", fld.Name) + } + + if fld.Notnull && exists == true && current == nil { + return fmt.Errorf("Can't be null field '%v'", fld.Name) + } + + if fld.Mandatory && exists == false { + return fmt.Errorf("Need to set field '%v'", fld.Name) } }