noth/surrealpatch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Encode all numbers as float64

Browse source 
All numbers in encoded keys are now encoded into float64, ensuring that decimal numbers are sorted alongside integers.

The maximum number able to be stored as a float64, without losing precision, is now (1<<53 - 1) - the same maximum number possible in javascript.
This commit is contained in:
Tobie Morgan Hitchcock 2016-09-10 00:21:58 +01:00
parent 0f420d2cae
commit 529ddbbec2
6 changed files with 194 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
util/keys/enc.go
View file

@ -99,12 +99,26 @@ func (e *encoder) Encode(items ...interface{}) {
e.w.Write(value) e.w.Write(value)
e.w.Write(bTERM) e.w.Write(bTERM)
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: case int, int8, int16, int32, int64:
e.w.Write(bNUMBER) e.w.Write(bNUMBER)
e.w.Write(value) e.w.Write(value)
e.w.Write(bTERM) e.w.Write(bTERM)
case uint, uint8, uint16, uint32, uint64:
e.w.Write(bNUMBER)
e.w.Write(value)
e.w.Write(bTERM)
case []time.Time:
e.w.Write(bARRAY)
for _, val := range value {
e.Encode(val)
}
e.w.Write(bTERM)
case []bool: case []bool:
e.w.Write(bARRAY) e.w.Write(bARRAY)

36
util/keys/full_test.go
View file

@ -22,10 +22,11 @@ type Full struct {
F bool // false F bool // false
S string // string S string // string
T time.Time // time.Time T time.Time // time.Time
N64 int64 // negative int64 NI64 int64 // negative int64
N32 int32 // negative int32 NI32 int32 // negative int32
N16 int16 // negative int16 NI16 int16 // negative int16
N8 int8 // negative int8 NI8 int8 // negative int8
NI int // negative int
I int // positive int I int // positive int
I8 int8 // positive int8 I8 int8 // positive int8
I16 int16 // positive int16 I16 int16 // positive int16
@ -36,16 +37,19 @@ type Full struct {
UI16 uint16 // positive uint16 UI16 uint16 // positive uint16
UI32 uint32 // positive uint32 UI32 uint32 // positive uint32
UI64 uint64 // positive uint64 UI64 uint64 // positive uint64
NF32 float32 // negative float32
NF64 float64 // negative float64 NF64 float64 // negative float64
NF32 float32 // negative float32
F32 float32 // positive float32 F32 float32 // positive float32
F64 float64 // positive float64 F64 float64 // positive float64
AB []bool // bool array AB []bool // bool array
AS []string // string array AS []string // string array
AT []time.Time // time array
AI []int // int array
AI8 []int8 // int8 array AI8 []int8 // int8 array
AI16 []int16 // int16 array AI16 []int16 // int16 array
AI32 []int32 // int32 array AI32 []int32 // int32 array
AI64 []int64 // int64 array AI64 []int64 // int64 array
AUI []uint // uint array
AUI8 []uint8 // uint8 array AUI8 []uint8 // uint8 array
AUI16 []uint16 // uint16 array AUI16 []uint16 // uint16 array
AUI32 []uint32 // uint32 array AUI32 []uint32 // uint32 array
@ -54,7 +58,7 @@ type Full struct {
AF64 []float64 // float64 array AF64 []float64 // float64 array
IN interface{} // interface{} IN interface{} // interface{}
IB interface{} // interface{} true IB interface{} // interface{} true
IF interface{} // interface{} talse IF interface{} // interface{} false
IT interface{} // interface{} time.Time IT interface{} // interface{} time.Time
II interface{} // interface{} number II interface{} // interface{} number
ID interface{} // interface{} double ID interface{} // interface{} double
@ -70,13 +74,13 @@ func (f *Full) Encode() []byte {
return encode( return encode(
f.N, f.B, f.F, f.S, f.T, f.N, f.B, f.F, f.S, f.T,
f.N64, f.N32, f.N16, f.N8, f.NI64, f.NI32, f.NI16, f.NI8, f.NI,
f.I, f.I8, f.I16, f.I32, f.I64, f.I, f.I8, f.I16, f.I32, f.I64,
f.UI, f.UI8, f.UI16, f.UI32, f.UI64, f.UI, f.UI8, f.UI16, f.UI32, f.UI64,
f.NF32, f.NF64, f.F32, f.F64, f.NF64, f.NF32, f.F32, f.F64,
f.AB, f.AS, f.AB, f.AS, f.AT,
f.AI8, f.AI16, f.AI32, f.AI64, f.AI, f.AI8, f.AI16, f.AI32, f.AI64,
f.AUI8, f.AUI16, f.AUI32, f.AUI64, f.AUI, f.AUI8, f.AUI16, f.AUI32, f.AUI64,
f.AF32, f.AF64, f.AF32, f.AF64,
f.IN, f.IB, f.IF, f.IT, f.II, f.ID, f.INA, f.AIN, f.IN, f.IB, f.IF, f.IT, f.II, f.ID, f.INA, f.AIN,
) )
@ -88,13 +92,13 @@ func (f *Full) Decode(data []byte) {
decode( decode(
data, data,
&f.N, &f.B, &f.F, &f.S, &f.T, &f.N, &f.B, &f.F, &f.S, &f.T,
&f.N64, &f.N32, &f.N16, &f.N8, &f.NI64, &f.NI32, &f.NI16, &f.NI8, &f.NI,
&f.I, &f.I8, &f.I16, &f.I32, &f.I64, &f.I, &f.I8, &f.I16, &f.I32, &f.I64,
&f.UI, &f.UI8, &f.UI16, &f.UI32, &f.UI64, &f.UI, &f.UI8, &f.UI16, &f.UI32, &f.UI64,
&f.NF32, &f.NF64, &f.F32, &f.F64, &f.NF64, &f.NF32, &f.F32, &f.F64,
&f.AB, &f.AS, &f.AB, &f.AS, &f.AT,
&f.AI8, &f.AI16, &f.AI32, &f.AI64, &f.AI, &f.AI8, &f.AI16, &f.AI32, &f.AI64,
&f.AUI8, &f.AUI16, &f.AUI32, &f.AUI64, &f.AUI, &f.AUI8, &f.AUI16, &f.AUI32, &f.AUI64,
&f.AF32, &f.AF64, &f.AF32, &f.AF64,
&f.IN, &f.IB, &f.IF, &f.IT, &f.II, &f.ID, &f.INA, &f.AIN, &f.IN, &f.IB, &f.IF, &f.IT, &f.II, &f.ID, &f.INA, &f.AIN,
) )

7
util/keys/keys.go
View file

@ -62,6 +62,13 @@ var (
bSUFFIX = []byte("\x08") bSUFFIX = []byte("\x08")
) )
const (
// MinNumber is the minimum number to be encoded and decoded
MinNumber = -1 << 53
// MaxNumber is the maximum number to be encoded and decoded
MaxNumber = 1<<53 - 1
)
// Key ... // Key ...
type Key interface { type Key interface {
String() string String() string

110
util/keys/keys_test.go
View file

@ -17,6 +17,7 @@ package keys
import ( import (
"bytes" "bytes"
"fmt" "fmt"
"math"
"testing" "testing"
"time" "time"
@ -171,49 +172,53 @@ func TestMain(t *testing.T) {
str: "Test key", str: "Test key",
new: &Full{}, new: &Full{},
obj: &Full{ obj: &Full{
N: nil, N: nil,
B: true, B: true,
F: false, F: false,
S: "Test", S: "Test",
T: clock, T: clock,
N64: -9223372036854775807, NI64: int64(MinNumber),
N32: -2147483647, NI32: math.MinInt32,
N16: -32767, NI16: math.MinInt16,
N8: -127, NI8: math.MinInt8,
I: 1, NI: -1,
I8: 127, I: 1,
I16: 32767, I8: math.MaxInt8,
I32: 2147483647, I16: math.MaxInt16,
I64: 9223372036854775807, I32: math.MaxInt32,
UI: 1, I64: int64(MaxNumber),
UI8: 255, UI: 1,
UI16: 65535, UI8: math.MaxUint8,
UI32: 4294967295, UI16: math.MaxUint16,
UI64: 18446744073709551615, UI32: math.MaxUint32,
// NF32: -0.00001, UI64: uint64(MaxNumber),
// NF64: -0.00002, NF64: -math.MaxFloat64,
// F32: 0.00001, NF32: -math.MaxFloat32,
// F64: 0.00002, F32: math.MaxFloat32,
F64: math.MaxFloat64,
AB: []bool{true, false}, AB: []bool{true, false},
AS: []string{"A", "B", "C"}, AS: []string{"A", "B", "C"},
AI8: []int8{127}, AT: []time.Time{clock, clock, clock},
AI16: []int16{32767}, AI: []int{1},
AI32: []int32{2147483647}, AI8: []int8{math.MaxInt8},
AI64: []int64{9223372036854775807}, AI16: []int16{math.MaxInt16},
AUI8: []uint8{127}, AI32: []int32{math.MaxInt32},
AUI16: []uint16{32767}, AI64: []int64{int64(MaxNumber)},
AUI32: []uint32{2147483647}, AUI: []uint{1},
AUI64: []uint64{9223372036854775807}, AUI8: []uint8{math.MaxUint8},
// AF32: []float32{0.1, 0.2, 0.3}, AUI16: []uint16{math.MaxUint16},
// AF64: []float64{0.1, 0.2, 0.3}, AUI32: []uint32{math.MaxUint32},
IN: "Test", AUI64: []uint64{uint64(MaxNumber)},
IB: true, AF32: []float32{1.1, 1.2, 1.3},
IF: false, AF64: []float64{1.1, 1.2, 1.3},
IT: clock, IN: "Test",
II: int64(19387), IB: true,
// ID: float64(183784.13413), IF: false,
// INA: []interface{}{true, false, nil, "Test", clock, int64(192), 0.1, 0.2, 0.3}, IT: clock,
// AIN: []interface{}{true, false, nil, "Test", clock, int64(192), int64(9223372036854775807), 0.1, 0.2, 0.3}, II: float64(19387),
ID: float64(183784.13413),
INA: []interface{}{true, false, nil, "Test", clock, float64(192), 1.1, 1.2, 1.3},
AIN: []interface{}{true, false, nil, "Test", clock, float64(192), float64(MaxNumber), 1.1, 1.2, 1.3, []interface{}{"Test"}},
}, },
}, },
} }
@ -235,13 +240,13 @@ func TestMain(t *testing.T) {
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 2}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 2},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 12}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 12},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 127}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 127},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int8(127)}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int8(math.MaxInt8)},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 32767}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 32767},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int16(32767)}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int16(math.MaxInt16)},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 2147483647}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 2147483647},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int32(2147483647)}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int32(math.MaxInt32)},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 9223372036854775807}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 9223372036854775807},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int64(9223372036854775807)}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int64(math.MaxInt64)},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: "A"}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: "A"},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: "B"}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: "B"},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: "Bb"}, &Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: "Bb"},
@ -280,11 +285,11 @@ func TestMain(t *testing.T) {
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 0, 127}}, &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 0, 127}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 0, 127}}, &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 0, 127}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 1, 127}}, &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 1, math.MaxInt8}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, 32767}}, &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, math.MaxInt16}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, 2147483647}}, &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, math.MaxInt32}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, 9223372036854775807}}, &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, MaxNumber}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, 9223372036854775807}}, &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, MaxNumber}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: Suffix}, &Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: Suffix},
} }
@ -369,10 +374,10 @@ func TestEncoding(t *testing.T) {
Convey(test.str, t, func() { Convey(test.str, t, func() {
enc := test.obj.Encode() enc := test.obj.Encode()
Printf("%s\n\n%#q\n\n%v\n\n", test.str, enc, enc)
test.new.Decode(enc) test.new.Decode(enc)
Convey("Key should encode and decode", func() { Convey("Key should encode and decode", func() {
Printf("%s\n\n%#q\n\n%v\n\n", test.str, enc, enc)
So(test.new, ShouldResemble, test.obj) So(test.new, ShouldResemble, test.obj)
}) })
@ -417,9 +422,8 @@ func TestSorting(t *testing.T) {
one := sorts[i-1].Encode() one := sorts[i-1].Encode()
two := sorts[i].Encode() two := sorts[i].Encode()
Printf("%#v\n%#v\n------\n%#v\n%#v\n------\n%#q\n%#q", sorts[i-1], sorts[i], one, two, one, two)
Convey("Key should sort before next key", func() { Convey("Key should sort before next key", func() {
Printf("%#v\n%#v\n------\n%#v\n%#v\n------\n%#q\n%#q", sorts[i-1], sorts[i], one, two, one, two)
So(string(one), ShouldBeLessThanOrEqualTo, string(two)) So(string(one), ShouldBeLessThanOrEqualTo, string(two))
}) })
}) })

13
util/keys/reader.go
View file

@ -127,11 +127,16 @@ func (r *reader) FindString() (val string) {
return return
} }
func (r *reader) FindNumber() (val int64) { func (r *reader) FindNumber() (val float64) {
if r.ReadNext(cNUMBER) { if r.ReadNext(cNUMBER) {
binary.Read(r.Reader, binary.BigEndian, &val) if r.ReadNext(cNILL) {
r.ReadNext(cTERM) binary.Read(r.Reader, binary.BigEndian, &val)
return val = 0 - val
r.ReadNext(cTERM)
} else if r.ReadNext(cBOOL) {
binary.Read(r.Reader, binary.BigEndian, &val)
r.ReadNext(cTERM)
}
} }
return return
} }

110
util/keys/writer.go
View file

@ -33,37 +33,99 @@ func newWriter(w io.Writer) *writer {
func (w *writer) Write(i interface{}) { func (w *writer) Write(i interface{}) {
switch v := i.(type) { switch v := i.(type) {
case []byte: case []byte:
w.Writer.Write(v) w.Writer.Write(v)
case string: case string:
w.Writer.Write([]byte(v)) w.Writer.Write([]byte(v))
case uint:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case uint8:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case uint16:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case uint32:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case uint64:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case int:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case int8:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case int16:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case int32:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case int64:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case float32:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case float64:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case time.Time: case time.Time:
binary.Write(w.Writer, binary.BigEndian, v.UnixNano()) binary.Write(w.Writer, binary.BigEndian, v.UnixNano())
case uint:
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
case uint8:
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
case uint16:
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
case uint32:
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
case uint64:
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
case int:
if v < 0 {
w.Write(bNILL)
binary.Write(w.Writer, binary.BigEndian, 0-float64(v))
} else {
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
}
case int8:
if v < 0 {
w.Write(bNILL)
binary.Write(w.Writer, binary.BigEndian, 0-float64(v))
} else {
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
}
case int16:
if v < 0 {
w.Write(bNILL)
binary.Write(w.Writer, binary.BigEndian, 0-float64(v))
} else {
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
}
case int32:
if v < 0 {
w.Write(bNILL)
binary.Write(w.Writer, binary.BigEndian, 0-float64(v))
} else {
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
}
case int64:
if v < 0 {
w.Write(bNILL)
binary.Write(w.Writer, binary.BigEndian, 0-float64(v))
} else {
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
}
case float32:
if v < 0 {
w.Write(bNILL)
binary.Write(w.Writer, binary.BigEndian, 0-float64(v))
} else {
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
}
case float64:
if v < 0 {
w.Write(bNILL)
binary.Write(w.Writer, binary.BigEndian, 0-float64(v))
} else {
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
}
} }
} }