noth/surrealpatch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Encode all numbers as float64

Browse source 
All numbers in encoded keys are now encoded into float64, ensuring that decimal numbers are sorted alongside integers.

The maximum number able to be stored as a float64, without losing precision, is now (1<<53 - 1) - the same maximum number possible in javascript.
This commit is contained in:
Tobie Morgan Hitchcock 2016-09-10 00:21:58 +01:00
parent 0f420d2cae
commit 529ddbbec2
6 changed files with 194 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
util/keys/enc.go
View file

@ -99,12 +99,26 @@ func (e *encoder) Encode(items ...interface{}) {
e.w.Write(value)
e.w.Write(bTERM)
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64:
case int, int8, int16, int32, int64:
e.w.Write(bNUMBER)
e.w.Write(value)
e.w.Write(bTERM)
case uint, uint8, uint16, uint32, uint64:
e.w.Write(bNUMBER)
e.w.Write(value)
e.w.Write(bTERM)
case []time.Time:
e.w.Write(bARRAY)
for _, val := range value {
e.Encode(val)
}
e.w.Write(bTERM)
case []bool:
e.w.Write(bARRAY)

36
util/keys/full_test.go
View file

@ -22,10 +22,11 @@ type Full struct {
F bool // false
S string // string
T time.Time // time.Time
N64 int64 // negative int64
N32 int32 // negative int32
N16 int16 // negative int16
N8 int8 // negative int8
NI64 int64 // negative int64
NI32 int32 // negative int32
NI16 int16 // negative int16
NI8 int8 // negative int8
NI int // negative int
I int // positive int
I8 int8 // positive int8
I16 int16 // positive int16
@ -36,16 +37,19 @@ type Full struct {
UI16 uint16 // positive uint16
UI32 uint32 // positive uint32
UI64 uint64 // positive uint64
NF32 float32 // negative float32
NF64 float64 // negative float64
NF32 float32 // negative float32
F32 float32 // positive float32
F64 float64 // positive float64
AB []bool // bool array
AS []string // string array
AT []time.Time // time array
AI []int // int array
AI8 []int8 // int8 array
AI16 []int16 // int16 array
AI32 []int32 // int32 array
AI64 []int64 // int64 array
AUI []uint // uint array
AUI8 []uint8 // uint8 array
AUI16 []uint16 // uint16 array
AUI32 []uint32 // uint32 array
@ -54,7 +58,7 @@ type Full struct {
AF64 []float64 // float64 array
IN interface{} // interface{}
IB interface{} // interface{} true
IF interface{} // interface{} talse
IF interface{} // interface{} false
IT interface{} // interface{} time.Time
II interface{} // interface{} number
ID interface{} // interface{} double
@ -70,13 +74,13 @@ func (f *Full) Encode() []byte {
return encode(
f.N, f.B, f.F, f.S, f.T,
f.N64, f.N32, f.N16, f.N8,
f.NI64, f.NI32, f.NI16, f.NI8, f.NI,
f.I, f.I8, f.I16, f.I32, f.I64,
f.UI, f.UI8, f.UI16, f.UI32, f.UI64,
f.NF32, f.NF64, f.F32, f.F64,
f.AB, f.AS,
f.AI8, f.AI16, f.AI32, f.AI64,
f.AUI8, f.AUI16, f.AUI32, f.AUI64,
f.NF64, f.NF32, f.F32, f.F64,
f.AB, f.AS, f.AT,
f.AI, f.AI8, f.AI16, f.AI32, f.AI64,
f.AUI, f.AUI8, f.AUI16, f.AUI32, f.AUI64,
f.AF32, f.AF64,
f.IN, f.IB, f.IF, f.IT, f.II, f.ID, f.INA, f.AIN,
)
@ -88,13 +92,13 @@ func (f *Full) Decode(data []byte) {
decode(
data,
&f.N, &f.B, &f.F, &f.S, &f.T,
&f.N64, &f.N32, &f.N16, &f.N8,
&f.NI64, &f.NI32, &f.NI16, &f.NI8, &f.NI,
&f.I, &f.I8, &f.I16, &f.I32, &f.I64,
&f.UI, &f.UI8, &f.UI16, &f.UI32, &f.UI64,
&f.NF32, &f.NF64, &f.F32, &f.F64,
&f.AB, &f.AS,
&f.AI8, &f.AI16, &f.AI32, &f.AI64,
&f.AUI8, &f.AUI16, &f.AUI32, &f.AUI64,
&f.NF64, &f.NF32, &f.F32, &f.F64,
&f.AB, &f.AS, &f.AT,
&f.AI, &f.AI8, &f.AI16, &f.AI32, &f.AI64,
&f.AUI, &f.AUI8, &f.AUI16, &f.AUI32, &f.AUI64,
&f.AF32, &f.AF64,
&f.IN, &f.IB, &f.IF, &f.IT, &f.II, &f.ID, &f.INA, &f.AIN,
)

7
util/keys/keys.go
View file

@ -62,6 +62,13 @@ var (
bSUFFIX = []byte("\x08")
)
const (
// MinNumber is the minimum number to be encoded and decoded
MinNumber = -1 << 53
// MaxNumber is the maximum number to be encoded and decoded
MaxNumber = 1<<53 - 1
)
// Key ...
type Key interface {
String() string

110
util/keys/keys_test.go
View file

@ -17,6 +17,7 @@ package keys
import (
"bytes"
"fmt"
"math"
"testing"
"time"
@ -171,49 +172,53 @@ func TestMain(t *testing.T) {
str: "Test key",
new: &Full{},
obj: &Full{
N: nil,
B: true,
F: false,
S: "Test",
T: clock,
N64: -9223372036854775807,
N32: -2147483647,
N16: -32767,
N8: -127,
I: 1,
I8: 127,
I16: 32767,
I32: 2147483647,
I64: 9223372036854775807,
UI: 1,
UI8: 255,
UI16: 65535,
UI32: 4294967295,
UI64: 18446744073709551615,
// NF32: -0.00001,
// NF64: -0.00002,
// F32: 0.00001,
// F64: 0.00002,
N: nil,
B: true,
F: false,
S: "Test",
T: clock,
NI64: int64(MinNumber),
NI32: math.MinInt32,
NI16: math.MinInt16,
NI8: math.MinInt8,
NI: -1,
I: 1,
I8: math.MaxInt8,
I16: math.MaxInt16,
I32: math.MaxInt32,
I64: int64(MaxNumber),
UI: 1,
UI8: math.MaxUint8,
UI16: math.MaxUint16,
UI32: math.MaxUint32,
UI64: uint64(MaxNumber),
NF64: -math.MaxFloat64,
NF32: -math.MaxFloat32,
F32: math.MaxFloat32,
F64: math.MaxFloat64,
AB: []bool{true, false},
AS: []string{"A", "B", "C"},
AI8: []int8{127},
AI16: []int16{32767},
AI32: []int32{2147483647},
AI64: []int64{9223372036854775807},
AUI8: []uint8{127},
AUI16: []uint16{32767},
AUI32: []uint32{2147483647},
AUI64: []uint64{9223372036854775807},
// AF32: []float32{0.1, 0.2, 0.3},
// AF64: []float64{0.1, 0.2, 0.3},
IN: "Test",
IB: true,
IF: false,
IT: clock,
II: int64(19387),
// ID: float64(183784.13413),
// INA: []interface{}{true, false, nil, "Test", clock, int64(192), 0.1, 0.2, 0.3},
// AIN: []interface{}{true, false, nil, "Test", clock, int64(192), int64(9223372036854775807), 0.1, 0.2, 0.3},
AT: []time.Time{clock, clock, clock},
AI: []int{1},
AI8: []int8{math.MaxInt8},
AI16: []int16{math.MaxInt16},
AI32: []int32{math.MaxInt32},
AI64: []int64{int64(MaxNumber)},
AUI: []uint{1},
AUI8: []uint8{math.MaxUint8},
AUI16: []uint16{math.MaxUint16},
AUI32: []uint32{math.MaxUint32},
AUI64: []uint64{uint64(MaxNumber)},
AF32: []float32{1.1, 1.2, 1.3},
AF64: []float64{1.1, 1.2, 1.3},
IN: "Test",
IB: true,
IF: false,
IT: clock,
II: float64(19387),
ID: float64(183784.13413),
INA: []interface{}{true, false, nil, "Test", clock, float64(192), 1.1, 1.2, 1.3},
AIN: []interface{}{true, false, nil, "Test", clock, float64(192), float64(MaxNumber), 1.1, 1.2, 1.3, []interface{}{"Test"}},
},
},
}
@ -235,13 +240,13 @@ func TestMain(t *testing.T) {
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 2},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 12},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 127},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int8(127)},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int8(math.MaxInt8)},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 32767},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int16(32767)},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int16(math.MaxInt16)},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 2147483647},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int32(2147483647)},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int32(math.MaxInt32)},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: 9223372036854775807},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int64(9223372036854775807)},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: int64(math.MaxInt64)},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: "A"},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: "B"},
&Thing{KV: "kv", NS: "ns", DB: "db", TB: "person", ID: "Bb"},
@ -280,11 +285,11 @@ func TestMain(t *testing.T) {
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 0, 127}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 0, 127}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 1, 127}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, 32767}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, 2147483647}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, 9223372036854775807}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, 9223372036854775807}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 1, math.MaxInt8}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, math.MaxInt16}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, math.MaxInt32}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, MaxNumber}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: []interface{}{"account:zymba", 2, MaxNumber}},
&Index{KV: "kv", NS: "ns", DB: "db", TB: "person", IX: "names", FD: Suffix},
}
@ -369,10 +374,10 @@ func TestEncoding(t *testing.T) {
Convey(test.str, t, func() {
enc := test.obj.Encode()
Printf("%s\n\n%#q\n\n%v\n\n", test.str, enc, enc)
test.new.Decode(enc)
Convey("Key should encode and decode", func() {
Printf("%s\n\n%#q\n\n%v\n\n", test.str, enc, enc)
So(test.new, ShouldResemble, test.obj)
})
@ -417,9 +422,8 @@ func TestSorting(t *testing.T) {
one := sorts[i-1].Encode()
two := sorts[i].Encode()
Printf("%#v\n%#v\n------\n%#v\n%#v\n------\n%#q\n%#q", sorts[i-1], sorts[i], one, two, one, two)
Convey("Key should sort before next key", func() {
Printf("%#v\n%#v\n------\n%#v\n%#v\n------\n%#q\n%#q", sorts[i-1], sorts[i], one, two, one, two)
So(string(one), ShouldBeLessThanOrEqualTo, string(two))
})
})

13
util/keys/reader.go
View file

@ -127,11 +127,16 @@ func (r *reader) FindString() (val string) {
return
}
func (r *reader) FindNumber() (val int64) {
func (r *reader) FindNumber() (val float64) {
if r.ReadNext(cNUMBER) {
binary.Read(r.Reader, binary.BigEndian, &val)
r.ReadNext(cTERM)
return
if r.ReadNext(cNILL) {
binary.Read(r.Reader, binary.BigEndian, &val)
val = 0 - val
r.ReadNext(cTERM)
} else if r.ReadNext(cBOOL) {
binary.Read(r.Reader, binary.BigEndian, &val)
r.ReadNext(cTERM)
}
}
return
}

110
util/keys/writer.go
View file

@ -33,37 +33,99 @@ func newWriter(w io.Writer) *writer {
func (w *writer) Write(i interface{}) {
switch v := i.(type) {
case []byte:
w.Writer.Write(v)
case string:
w.Writer.Write([]byte(v))
case uint:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case uint8:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case uint16:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case uint32:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case uint64:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case int:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case int8:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case int16:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case int32:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case int64:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case float32:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case float64:
binary.Write(w.Writer, binary.BigEndian, int64(v))
case time.Time:
binary.Write(w.Writer, binary.BigEndian, v.UnixNano())
case uint:
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
case uint8:
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
case uint16:
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
case uint32:
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
case uint64:
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
case int:
if v < 0 {
w.Write(bNILL)
binary.Write(w.Writer, binary.BigEndian, 0-float64(v))
} else {
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
}
case int8:
if v < 0 {
w.Write(bNILL)
binary.Write(w.Writer, binary.BigEndian, 0-float64(v))
} else {
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
}
case int16:
if v < 0 {
w.Write(bNILL)
binary.Write(w.Writer, binary.BigEndian, 0-float64(v))
} else {
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
}
case int32:
if v < 0 {
w.Write(bNILL)
binary.Write(w.Writer, binary.BigEndian, 0-float64(v))
} else {
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
}
case int64:
if v < 0 {
w.Write(bNILL)
binary.Write(w.Writer, binary.BigEndian, 0-float64(v))
} else {
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
}
case float32:
if v < 0 {
w.Write(bNILL)
binary.Write(w.Writer, binary.BigEndian, 0-float64(v))
} else {
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
}
case float64:
if v < 0 {
w.Write(bNILL)
binary.Write(w.Writer, binary.BigEndian, 0-float64(v))
} else {
w.Write(bBOOL)
binary.Write(w.Writer, binary.BigEndian, float64(v))
}
}
}