From 35047ce04c4ff232960130c9a1cbf582648da174 Mon Sep 17 00:00:00 2001 From: Tobie Morgan Hitchcock Date: Fri, 20 Apr 2018 23:51:42 +0100 Subject: [PATCH] Improve remote array record fetching --- db/fetch.go | 36 +++++++++++++++++++++++++++++++----- sql/scanner.go | 2 +- util/data/data.go | 35 +++++++++++++++++++---------------- util/data/data_test.go | 12 ++++++------ 4 files changed, 57 insertions(+), 28 deletions(-) diff --git a/db/fetch.go b/db/fetch.go index c8a48e7e..e831a7c1 100644 --- a/db/fetch.go +++ b/db/fetch.go @@ -86,6 +86,9 @@ func (e *executor) fetch(ctx context.Context, val interface{}, doc *data.Doc) (o doc.Fetch(func(key string, val interface{}) interface{} { switch res := val.(type) { + case []interface{}: + val, _ = e.fetchArray(ctx, res, doc) + return val case *sql.Thing: val, _ = e.fetchThing(ctx, res, doc) return val @@ -106,6 +109,9 @@ func (e *executor) fetch(ctx context.Context, val interface{}, doc *data.Doc) (o obj.Fetch(func(key string, val interface{}) interface{} { switch res := val.(type) { + case []interface{}: + val, _ = e.fetchArray(ctx, res, doc) + return val case *sql.Thing: val, _ = e.fetchThing(ctx, res, doc) return val @@ -328,11 +334,12 @@ func (e *executor) fetchPaths(ctx context.Context, doc *data.Doc, exprs ...sql.E func (e *executor) fetchThing(ctx context.Context, val *sql.Thing, doc *data.Doc) (interface{}, error) { res, err := e.executeSelect(ctx, &sql.SelectStatement{ - KV: cnf.Settings.DB.Base, - NS: ctx.Value(ctxKeyNs).(string), - DB: ctx.Value(ctxKeyDb).(string), - Expr: []*sql.Field{{Expr: &sql.All{}}}, - What: []sql.Expr{val}, + KV: cnf.Settings.DB.Base, + NS: ctx.Value(ctxKeyNs).(string), + DB: ctx.Value(ctxKeyDb).(string), + Expr: []*sql.Field{{Expr: &sql.All{}}}, + What: []sql.Expr{val}, + Parallel: 1, }) if err != nil { @@ -347,6 +354,25 @@ func (e *executor) fetchThing(ctx context.Context, val *sql.Thing, doc *data.Doc } +func (e *executor) fetchArray(ctx context.Context, val []interface{}, doc *data.Doc) (interface{}, error) { + + res, err := e.executeSelect(ctx, &sql.SelectStatement{ + KV: cnf.Settings.DB.Base, + NS: ctx.Value(ctxKeyNs).(string), + DB: ctx.Value(ctxKeyDb).(string), + Expr: []*sql.Field{{Expr: &sql.All{}}}, + What: []sql.Expr{val}, + Parallel: 1, + }) + + if err != nil { + return nil, err + } + + return res, nil + +} + func (e *executor) fetchLimit(ctx context.Context, val sql.Expr) (int, error) { v, err := e.fetch(ctx, val, nil) diff --git a/sql/scanner.go b/sql/scanner.go index 8878f4ab..f6b5919a 100644 --- a/sql/scanner.go +++ b/sql/scanner.go @@ -1050,7 +1050,7 @@ func isThingChar(ch rune) bool { // isExprsChar returns true if the rune is allowed in a IDENT. func isExprsChar(ch rune) bool { - return isLetter(ch) || isNumber(ch) || ch == '.' || ch == '_' || ch == '*' || ch == '[' || ch == ']' + return isLetter(ch) || isNumber(ch) || ch == '.' || ch == '_' || ch == '*' || ch == '[' || ch == '$' || ch == ']' } // eof represents a marker rune for the end of the reader. diff --git a/util/data/data.go b/util/data/data.go index c20a8859..a67f3654 100644 --- a/util/data/data.go +++ b/util/data/data.go @@ -58,6 +58,11 @@ func Consume(input interface{}) *Doc { return &Doc{data: input} } +// Consume converts a GO interface into a data object. +func ConsumeWithFetch(input interface{}, fetcher Fetcher) *Doc { + return &Doc{data: input, call: fetcher} +} + // Data returns the internal data object as an interface. func (d *Doc) Data() interface{} { return d.data @@ -428,7 +433,7 @@ func (d *Doc) Exists(path ...string) bool { if d.call != nil && len(path[k+1:]) > 0 { c[0] = d.call(p, c[0]) } - return Consume(c[0]).Exists(path[k+1:]...) + return ConsumeWithFetch(c[0], d.call).Exists(path[k+1:]...) } if r == many { @@ -436,7 +441,7 @@ func (d *Doc) Exists(path ...string) bool { if d.call != nil && len(path[k+1:]) > 0 { v = d.call(p, v) } - if !Consume(v).Exists(path[k+1:]...) { + if !ConsumeWithFetch(v, d.call).Exists(path[k+1:]...) { return false } } @@ -516,7 +521,7 @@ func (d *Doc) Get(path ...string) *Doc { if d.call != nil && len(path[k+1:]) > 0 { c[0] = d.call(p, c[0]) } - return Consume(c[0]).Get(path[k+1:]...) + return ConsumeWithFetch(c[0], d.call).Get(path[k+1:]...) } if r == many { @@ -525,10 +530,8 @@ func (d *Doc) Get(path ...string) *Doc { if d.call != nil && len(path[k+1:]) > 0 { v = d.call(p, v) } - res := Consume(v).Get(path[k+1:]...) - if res.data != nil { - out = append(out, res.data) - } + res := ConsumeWithFetch(v, d.call).Get(path[k+1:]...) + out = append(out, res.data) } return &Doc{data: out} } @@ -608,7 +611,7 @@ func (d *Doc) Set(value interface{}, path ...string) (*Doc, error) { a[i[0]] = value object = a[i[0]] } else { - return Consume(a[i[0]]).Set(value, path[k+1:]...) + return ConsumeWithFetch(a[i[0]], d.call).Set(value, path[k+1:]...) } } @@ -619,7 +622,7 @@ func (d *Doc) Set(value interface{}, path ...string) (*Doc, error) { a[i[j]] = value out = append(out, value) } else { - res, _ := Consume(v).Set(value, path[k+1:]...) + res, _ := ConsumeWithFetch(v, d.call).Set(value, path[k+1:]...) if res.data != nil { out = append(out, res.data) } @@ -699,7 +702,7 @@ func (d *Doc) Del(path ...string) error { d.Set(c, path[:len(path)-1]...) } else { if len(c) != 0 { - return Consume(c[0]).Del(path[k+1:]...) + return ConsumeWithFetch(c[0], d.call).Del(path[k+1:]...) } } } @@ -709,7 +712,7 @@ func (d *Doc) Del(path ...string) error { d.Set(c, path[:len(path)-1]...) } else { for _, v := range c { - Consume(v).Del(path[k+1:]...) + ConsumeWithFetch(v, d.call).Del(path[k+1:]...) } break } @@ -766,7 +769,7 @@ func (d *Doc) ArrayAdd(value interface{}, path ...string) (*Doc, error) { } else { for _, v := range a { if reflect.DeepEqual(v, value) { - return Consume(a), nil + return ConsumeWithFetch(a, d.call), nil } } a = append(a, value) @@ -974,7 +977,7 @@ func (d *Doc) each(exec Iterator, prev []string) error { var keep []string keep = append(keep, prev...) keep = append(keep, k) - Consume(v).each(exec, keep) + ConsumeWithFetch(v, d.call).each(exec, keep) } return nil } @@ -989,7 +992,7 @@ func (d *Doc) each(exec Iterator, prev []string) error { var keep []string keep = append(keep, prev...) keep = append(keep, fmt.Sprintf("[%d]", i)) - Consume(v).each(exec, keep) + ConsumeWithFetch(v, d.call).each(exec, keep) } return nil } @@ -1072,7 +1075,7 @@ func (d *Doc) walk(exec Iterator, prev []string, path ...string) error { keep = append(keep, prev...) keep = append(keep, path[:k]...) keep = append(keep, fmt.Sprintf("[%d]", i[0])) - return Consume(c[0]).walk(exec, keep, path[k+1:]...) + return ConsumeWithFetch(c[0], d.call).walk(exec, keep, path[k+1:]...) } } @@ -1091,7 +1094,7 @@ func (d *Doc) walk(exec Iterator, prev []string, path ...string) error { keep = append(keep, prev...) keep = append(keep, path[:k]...) keep = append(keep, fmt.Sprintf("[%d]", i[j])) - if err := Consume(v).walk(exec, keep, path[k+1:]...); err != nil { + if err := ConsumeWithFetch(v, d.call).walk(exec, keep, path[k+1:]...); err != nil { return err } } diff --git a/util/data/data_test.go b/util/data/data_test.go index 0b02054c..2f070aa4 100644 --- a/util/data/data_test.go +++ b/util/data/data_test.go @@ -872,36 +872,36 @@ func TestOperations(t *testing.T) { err := doc.Del("the.item.arrays.0.id") So(err, ShouldBeNil) So(doc.Get("the.item.arrays.0.id").Data(), ShouldResemble, nil) - So(doc.Get("the.item.arrays.*.id").Data(), ShouldResemble, []interface{}{"ID2"}) + So(doc.Get("the.item.arrays.*.id").Data(), ShouldResemble, []interface{}{nil, "ID2"}) }) Convey("Can't del array → 5 → key", t, func() { err := doc.Del("the.item.arrays.5.id") So(err, ShouldBeNil) - So(doc.Get("the.item.arrays.*.id").Data(), ShouldResemble, []interface{}{"ID2"}) + So(doc.Get("the.item.arrays.*.id").Data(), ShouldResemble, []interface{}{nil, "ID2"}) }) Convey("Can del array → * → key", t, func() { err := doc.Del("the.item.arrays.*.id") So(err, ShouldBeNil) So(doc.Get("the.item.arrays.0.id").Data(), ShouldResemble, nil) - So(doc.Get("the.item.arrays.*.id").Data(), ShouldResemble, []interface{}{}) + So(doc.Get("the.item.arrays.*.id").Data(), ShouldResemble, []interface{}{nil, nil}) }) // ---------------------------------------------------------------------------------------------------- Convey("Can get object → key", t, func() { - So(doc.Get("the.item.arrays.*.one").Data(), ShouldResemble, []interface{}{"one"}) + So(doc.Get("the.item.arrays.*.one").Data(), ShouldResemble, []interface{}{"one", nil}) }) Convey("Can get object → key", t, func() { - So(doc.Get("the.item.arrays.*.two").Data(), ShouldResemble, []interface{}{"two"}) + So(doc.Get("the.item.arrays.*.two").Data(), ShouldResemble, []interface{}{nil, "two"}) }) // ---------------------------------------------------------------------------------------------------- Convey("Can get array → * → object → !", t, func() { - So(doc.Get("the.item.arrays.*.selected.none").Data(), ShouldResemble, []interface{}{}) + So(doc.Get("the.item.arrays.*.selected.none").Data(), ShouldResemble, []interface{}{nil, nil}) }) Convey("Can set array → * → object → !", t, func() {