From 529ddbbec2c7ae8d41e7f8a9d14f4f4e0d341b2c Mon Sep 17 00:00:00 2001 From: Tobie Morgan Hitchcock Date: Sat, 10 Sep 2016 00:21:58 +0100 Subject: [PATCH] Encode all numbers as float64 All numbers in encoded keys are now encoded into float64, ensuring that decimal numbers are sorted alongside integers. The maximum number able to be stored as a float64, without losing precision, is now (1<<53 - 1) - the same maximum number possible in javascript. --- util/keys/enc.go | 16 +++++- util/keys/full_test.go | 36 ++++++++------ util/keys/keys.go | 7 +++ util/keys/keys_test.go | 110 +++++++++++++++++++++-------------------- util/keys/reader.go | 13 +++-- util/keys/writer.go | 110 ++++++++++++++++++++++++++++++++--------- 6 files changed, 194 insertions(+), 98 deletions(-) diff --git a/util/keys/enc.go b/util/keys/enc.go index 95f7309e..88d9dc1d 100644 --- a/util/keys/enc.go +++ b/util/keys/enc.go @@ -99,12 +99,26 @@ func (e *encoder) Encode(items ...interface{}) { e.w.Write(value) e.w.Write(bTERM) - case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + case int, int8, int16, int32, int64: e.w.Write(bNUMBER) e.w.Write(value) e.w.Write(bTERM) + case uint, uint8, uint16, uint32, uint64: + + e.w.Write(bNUMBER) + e.w.Write(value) + e.w.Write(bTERM) + + case []time.Time: + + e.w.Write(bARRAY) + for _, val := range value { + e.Encode(val) + } + e.w.Write(bTERM) + case []bool: e.w.Write(bARRAY) diff --git a/util/keys/full_test.go b/util/keys/full_test.go index ec5fbc62..ff8d79ed 100644 --- a/util/keys/full_test.go +++ b/util/keys/full_test.go @@ -22,10 +22,11 @@ type Full struct { F bool // false S string // string T time.Time // time.Time - N64 int64 // negative int64 - N32 int32 // negative int32 - N16 int16 // negative int16 - N8 int8 // negative int8 + NI64 int64 // negative int64 + NI32 int32 // negative int32 + NI16 int16 // negative int16 + NI8 int8 // negative int8 + NI int // negative int I int // positive int I8 int8 // positive int8 I16 int16 // positive int16 @@ -36,16 +37,19 @@ type Full struct { UI16 uint16 // positive uint16 UI32 uint32 // positive uint32 UI64 uint64 // positive uint64 - NF32 float32 // negative float32 NF64 float64 // negative float64 + NF32 float32 // negative float32 F32 float32 // positive float32 F64 float64 // positive float64 AB []bool // bool array AS []string // string array + AT []time.Time // time array + AI []int // int array AI8 []int8 // int8 array AI16 []int16 // int16 array AI32 []int32 // int32 array AI64 []int64 // int64 array + AUI []uint // uint array AUI8 []uint8 // uint8 array AUI16 []uint16 // uint16 array AUI32 []uint32 // uint32 array @@ -54,7 +58,7 @@ type Full struct { AF64 []float64 // float64 array IN interface{} // interface{} IB interface{} // interface{} true - IF interface{} // interface{} talse + IF interface{} // interface{} false IT interface{} // interface{} time.Time II interface{} // interface{} number ID interface{} // interface{} double @@ -70,13 +74,13 @@ func (f *Full) Encode() []byte { return encode( f.N, f.B, f.F, f.S, f.T, - f.N64, f.N32, f.N16, f.N8, + f.NI64, f.NI32, f.NI16, f.NI8, f.NI, f.I, f.I8, f.I16, f.I32, f.I64, f.UI, f.UI8, f.UI16, f.UI32, f.UI64, - f.NF32, f.NF64, f.F32, f.F64, - f.AB, f.AS, - f.AI8, f.AI16, f.AI32, f.AI64, - f.AUI8, f.AUI16, f.AUI32, f.AUI64, + f.NF64, f.NF32, f.F32, f.F64, + f.AB, f.AS, f.AT, + f.AI, f.AI8, f.AI16, f.AI32, f.AI64, + f.AUI, f.AUI8, f.AUI16, f.AUI32, f.AUI64, f.AF32, f.AF64, f.IN, f.IB, f.IF, f.IT, f.II, f.ID, f.INA, f.AIN, ) @@ -88,13 +92,13 @@ func (f *Full) Decode(data []byte) { decode( data, &f.N, &f.B, &f.F, &f.S, &f.T, - &f.N64, &f.N32, &f.N16, &f.N8, + &f.NI64, &f.NI32, &f.NI16, &f.NI8, &f.NI, &f.I, &f.I8, &f.I16, &f.I32, &f.I64, &f.UI, &f.UI8, &f.UI16, &f.UI32, &f.UI64, - &f.NF32, &f.NF64, &f.F32, &f.F64, - &f.AB, &f.AS, - &f.AI8, &f.AI16, &f.AI32, &f.AI64, - &f.AUI8, &f.AUI16, &f.AUI32, &f.AUI64, + &f.NF64, &f.NF32, &f.F32, &f.F64, + &f.AB, &f.AS, &f.AT, + &f.AI, &f.AI8, &f.AI16, &f.AI32, &f.AI64, + &f.AUI, &f.AUI8, &f.AUI16, &f.AUI32, &f.AUI64, &f.AF32, &f.AF64, &f.IN, &f.IB, &f.IF, &f.IT, &f.II, &f.ID, &f.INA, &f.AIN, ) diff --git a/util/keys/keys.go b/util/keys/keys.go index e1836275..fbe83145 100644 --- a/util/keys/keys.go +++ b/util/keys/keys.go @@ -62,6 +62,13 @@ var ( bSUFFIX = []byte("\x08") ) +const ( + // MinNumber is the minimum number to be encoded and decoded + MinNumber = -1 << 53 + // MaxNumber is the maximum number to be encoded and decoded + MaxNumber = 1<<53 - 1 +) + // Key ... type Key interface { String() string diff --git a/util/keys/keys_test.go b/util/keys/keys_test.go index fff9b83e..7fcf3f62 100644 --- a/util/keys/keys_test.go +++ b/util/keys/keys_test.go @@ -17,6 +17,7 @@ package keys import ( "bytes" "fmt" + "math" "testing" "time" @@ -171,49 +172,53 @@ func TestMain(t *testing.T) { str: "Test key", new: &Full{}, obj: &Full{ - N: nil, - B: true, - F: false, - S: "Test", - T: clock, - N64: -9223372036854775807, - N32: -2147483647, - N16: -32767, - N8: -127, - I: 1, - I8: 127, - I16: 32767, - I32: 2147483647, - I64: 9223372036854775807, - UI: 1, - UI8: 255, - UI16: 65535, - UI32: 4294967295, - UI64: 18446744073709551615, - // NF32: -0.00001, - // NF64: -0.00002, - // F32: 0.00001, - // F64: 0.00002, + N: nil, + B: true, + F: false, + S: "Test", + T: clock, + NI64: int64(MinNumber), + NI32: math.MinInt32, + NI16: math.MinInt16, + NI8: math.MinInt8, + NI: -1, + I: 1, + I8: math.MaxInt8, + I16: math.MaxInt16, + I32: math.MaxInt32, + I64: int64(MaxNumber), + UI: 1, + UI8: math.MaxUint8, + UI16: math.MaxUint16, + UI32: math.MaxUint32, + UI64: uint64(MaxNumber), + NF64: -math.MaxFloat64, + NF32: -math.MaxFloat32, + F32: math.MaxFloat32, + F64: math.MaxFloat64, AB: []bool{true, false}, AS: []string{"A", "B", "C"}, - AI8: []int8{127}, - AI16: []int16{32767}, - AI32: []int32{2147483647}, - AI64: []int64{9223372036854775807}, - AUI8: []uint8{127}, - AUI16: []uint16{32767}, - AUI32: []uint32{2147483647}, - AUI64: []uint64{9223372036854775807}, - // AF32: []float32{0.1, 0.2, 0.3}, - // AF64: []float64{0.1, 0.2, 0.3}, - IN: "Test", - IB: true, - IF: false, - IT: clock, - II: int64(19387), - // ID: float64(183784.13413), - // INA: []interface{}{true, false, nil, "Test", clock, int64(192), 0.1, 0.2, 0.3}, - // AIN: []interface{}{true, false, nil, "Test", clock, int64(192), int64(9223372036854775807), 0.1, 0.2, 0.3}, + AT: []time.Time{clock, clock, clock}, + AI: []int{1}, + AI8: []int8{math.MaxInt8}, + AI16: []int16{math.MaxInt16}, + AI32: []int32{math.MaxInt32}, + AI64: []int64{int64(MaxNumber)}, + AUI: []uint{1}, + AUI8: []uint8{math.MaxUint8}, + AUI16: []uint16{math.MaxUint16}, + AUI32: []uint32{math.MaxUint32}, + AUI64: []uint64{uint64(MaxNumber)}, + AF32: []float32{1.1, 1.2, 1.3}, + AF64: []float64{1.1, 1.2, 1.3}, + IN: "Test", + IB: true, + IF: false, + IT: clock, + II: float64(19387), + ID: float64(183784.13413), + INA: []interface{}{true, false, nil, "Test", clock, float64(192), 1.1, 1.2, 1.3}, + AIN: []interface{}{true, false, nil, "Test", clock, float64(192), float64(MaxNumber), 1.1, 1.2, 1.3, []interface{}{"Test"}}, }, }, } @@ -235,13 +240,13 @@ func TestMain(t *testing.T) { &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 2}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 12}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 127}, - &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int8(127)}, + &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int8(math.MaxInt8)}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 32767}, - &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int16(32767)}, + &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int16(math.MaxInt16)}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 2147483647}, - &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int32(2147483647)}, + &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int32(math.MaxInt32)}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 9223372036854775807}, - &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int64(9223372036854775807)}, + &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int64(math.MaxInt64)}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: "A"}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: "B"}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: "Bb"}, @@ -280,11 +285,11 @@ func TestMain(t *testing.T) { &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 0, 127}}, &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 0, 127}}, - &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 1, 127}}, - &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, 32767}}, - &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, 2147483647}}, - &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, 9223372036854775807}}, - &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, 9223372036854775807}}, + &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 1, math.MaxInt8}}, + &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, math.MaxInt16}}, + &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, math.MaxInt32}}, + &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, MaxNumber}}, + &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, MaxNumber}}, &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: Suffix}, } @@ -369,10 +374,10 @@ func TestEncoding(t *testing.T) { Convey(test.str, t, func() { enc := test.obj.Encode() - Printf("%s\n\n%#q\n\n%v\n\n", test.str, enc, enc) test.new.Decode(enc) Convey("Key should encode and decode", func() { + Printf("%s\n\n%#q\n\n%v\n\n", test.str, enc, enc) So(test.new, ShouldResemble, test.obj) }) @@ -417,9 +422,8 @@ func TestSorting(t *testing.T) { one := sorts[i-1].Encode() two := sorts[i].Encode() - Printf("%#v\n%#v\n------\n%#v\n%#v\n------\n%#q\n%#q", sorts[i-1], sorts[i], one, two, one, two) - Convey("Key should sort before next key", func() { + Printf("%#v\n%#v\n------\n%#v\n%#v\n------\n%#q\n%#q", sorts[i-1], sorts[i], one, two, one, two) So(string(one), ShouldBeLessThanOrEqualTo, string(two)) }) }) diff --git a/util/keys/reader.go b/util/keys/reader.go index e3d5afa8..b364d847 100644 --- a/util/keys/reader.go +++ b/util/keys/reader.go @@ -127,11 +127,16 @@ func (r *reader) FindString() (val string) { return } -func (r *reader) FindNumber() (val int64) { +func (r *reader) FindNumber() (val float64) { if r.ReadNext(cNUMBER) { - binary.Read(r.Reader, binary.BigEndian, &val) - r.ReadNext(cTERM) - return + if r.ReadNext(cNILL) { + binary.Read(r.Reader, binary.BigEndian, &val) + val = 0 - val + r.ReadNext(cTERM) + } else if r.ReadNext(cBOOL) { + binary.Read(r.Reader, binary.BigEndian, &val) + r.ReadNext(cTERM) + } } return } diff --git a/util/keys/writer.go b/util/keys/writer.go index 07d40a7f..76aa7175 100644 --- a/util/keys/writer.go +++ b/util/keys/writer.go @@ -33,37 +33,99 @@ func newWriter(w io.Writer) *writer { func (w *writer) Write(i interface{}) { switch v := i.(type) { + case []byte: w.Writer.Write(v) + case string: w.Writer.Write([]byte(v)) - case uint: - binary.Write(w.Writer, binary.BigEndian, int64(v)) - case uint8: - binary.Write(w.Writer, binary.BigEndian, int64(v)) - case uint16: - binary.Write(w.Writer, binary.BigEndian, int64(v)) - case uint32: - binary.Write(w.Writer, binary.BigEndian, int64(v)) - case uint64: - binary.Write(w.Writer, binary.BigEndian, int64(v)) - case int: - binary.Write(w.Writer, binary.BigEndian, int64(v)) - case int8: - binary.Write(w.Writer, binary.BigEndian, int64(v)) - case int16: - binary.Write(w.Writer, binary.BigEndian, int64(v)) - case int32: - binary.Write(w.Writer, binary.BigEndian, int64(v)) - case int64: - binary.Write(w.Writer, binary.BigEndian, int64(v)) - case float32: - binary.Write(w.Writer, binary.BigEndian, int64(v)) - case float64: - binary.Write(w.Writer, binary.BigEndian, int64(v)) + case time.Time: binary.Write(w.Writer, binary.BigEndian, v.UnixNano()) + case uint: + w.Write(bBOOL) + binary.Write(w.Writer, binary.BigEndian, float64(v)) + + case uint8: + w.Write(bBOOL) + binary.Write(w.Writer, binary.BigEndian, float64(v)) + + case uint16: + w.Write(bBOOL) + binary.Write(w.Writer, binary.BigEndian, float64(v)) + + case uint32: + w.Write(bBOOL) + binary.Write(w.Writer, binary.BigEndian, float64(v)) + + case uint64: + w.Write(bBOOL) + binary.Write(w.Writer, binary.BigEndian, float64(v)) + + case int: + if v < 0 { + w.Write(bNILL) + binary.Write(w.Writer, binary.BigEndian, 0-float64(v)) + } else { + w.Write(bBOOL) + binary.Write(w.Writer, binary.BigEndian, float64(v)) + } + + case int8: + if v < 0 { + w.Write(bNILL) + binary.Write(w.Writer, binary.BigEndian, 0-float64(v)) + } else { + w.Write(bBOOL) + binary.Write(w.Writer, binary.BigEndian, float64(v)) + } + + case int16: + if v < 0 { + w.Write(bNILL) + binary.Write(w.Writer, binary.BigEndian, 0-float64(v)) + } else { + w.Write(bBOOL) + binary.Write(w.Writer, binary.BigEndian, float64(v)) + } + + case int32: + if v < 0 { + w.Write(bNILL) + binary.Write(w.Writer, binary.BigEndian, 0-float64(v)) + } else { + w.Write(bBOOL) + binary.Write(w.Writer, binary.BigEndian, float64(v)) + } + + case int64: + if v < 0 { + w.Write(bNILL) + binary.Write(w.Writer, binary.BigEndian, 0-float64(v)) + } else { + w.Write(bBOOL) + binary.Write(w.Writer, binary.BigEndian, float64(v)) + } + + case float32: + if v < 0 { + w.Write(bNILL) + binary.Write(w.Writer, binary.BigEndian, 0-float64(v)) + } else { + w.Write(bBOOL) + binary.Write(w.Writer, binary.BigEndian, float64(v)) + } + + case float64: + if v < 0 { + w.Write(bNILL) + binary.Write(w.Writer, binary.BigEndian, 0-float64(v)) + } else { + w.Write(bBOOL) + binary.Write(w.Writer, binary.BigEndian, float64(v)) + } + } }