From 7a3355bff03ec3f148cbd9115a72315843fa3db7 Mon Sep 17 00:00:00 2001 From: Tobie Morgan Hitchcock Date: Fri, 12 Jan 2018 10:56:21 +0000 Subject: [PATCH] Enable customisable file size policies for storage backends --- cli/flags.go | 14 +++++++++++--- cli/setup.go | 8 ++++++++ cli/start.go | 1 + cnf/cnf.go | 1 + db/db_test.go | 2 ++ kvs/rixxdb/main.go | 12 +++++++++--- 6 files changed, 32 insertions(+), 6 deletions(-) diff --git a/cli/flags.go b/cli/flags.go index 04d676d2..6c2a4e85 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -15,10 +15,11 @@ package cli var flags = map[string]string{ - "db": `Database configuration path used for storing data. Available backend stores are memory, file, s3, gcs, rixxdb, or dendrodb. (default "memory").`, + "db": `Database configuration path used for storing data. Available backend stores are memory, file, logr, s3, gcs, or dendrodb. (default "memory").`, "key": `Encryption key to use for intra-cluster communications, and on-disk encryption. For AES-128 encryption use a 16 bit key, for AES-192 encryption use a 24 bit key, and for AES-256 encryption use a 32 bit key.`, - "sync": `A time duration to use when syncing data to persistent storage. To sync data with every write specify '0', otherwise the data will be persisted asynchronously after the specified duration.`, - "shrink": `A time duration to use when shrinking data on persistent storage. To shrink data asynchronously after a repeating period of time, specify a duration.`, + "size": `A size in MB which determines the minimum or maximum file size for streaming data file storage. This is used for specifying maximum cached data sizes when using remote streaming storage. (default "5")`, + "sync": `A time duration to use when syncing data to persistent storage. To sync data with every write specify '0', otherwise the data will be persisted asynchronously after the specified duration. (default "0s")`, + "shrink": `A time duration to use when shrinking data on persistent storage. To shrink data asynchronously after a repeating period of time, specify a duration. Disabled by default. (default "0s")`, "join": `A comma-separated list of addresses to use when a new node is joining an existing cluster. For the first node in a cluster, --join should NOT be specified.`, } @@ -36,11 +37,18 @@ var usage = map[string][]string{ "--key 1hg7dbrma8ghe5473kghvie6", "--key 1hg7dbrma8ghe5473kghvie64jgi3ph4", }, + "size": { + "--db-size 30", + }, "sync": { "--db-sync 0", "--db-sync 5s", "--db-sync 1m", }, + "shrink": { + "--db-shrink 30m", + "--db-shrink 24h", + }, "join": { "--join 10.0.0.1", "--join 10.0.0.1:33693", diff --git a/cli/setup.go b/cli/setup.go index b08be38f..9fa5d26a 100644 --- a/cli/setup.go +++ b/cli/setup.go @@ -61,6 +61,14 @@ func setup() { } } + if opts.DB.Proc.Size == 0 { + opts.DB.Proc.Size = 5 + } + + if opts.DB.Proc.Size < 0 { + log.Fatal("Specify a valid data file size policy. Valid sizes are greater than 0 and are specified in MB.") + } + if strings.HasPrefix(opts.DB.Cert.CA, "-----") { var err error var doc *os.File diff --git a/cli/start.go b/cli/start.go index 62ba810d..13c214d4 100644 --- a/cli/start.go +++ b/cli/start.go @@ -94,6 +94,7 @@ func init() { startCmd.PersistentFlags().StringVar(&opts.DB.Cert.Crt, "db-crt", "", "Path to the certificate file used to connect to the remote database.") startCmd.PersistentFlags().StringVar(&opts.DB.Cert.Key, "db-key", "", "Path to the private key file used to connect to the remote database.") startCmd.PersistentFlags().StringVar(&opts.DB.Path, "db-path", "", flag("db")) + startCmd.PersistentFlags().IntVar(&opts.DB.Proc.Size, "db-size", 0, flag("size")) startCmd.PersistentFlags().DurationVar(&opts.DB.Proc.Sync, "db-sync", 0, flag("sync")) startCmd.PersistentFlags().DurationVar(&opts.DB.Proc.Shrink, "db-shrink", 0, flag("shrink")) diff --git a/cnf/cnf.go b/cnf/cnf.go index 4dac4ac8..ac4740b9 100644 --- a/cnf/cnf.go +++ b/cnf/cnf.go @@ -84,6 +84,7 @@ type Options struct { Port string // Surreal port to connect to Base string // Base key to use in KV stores Proc struct { + Size int // Policy for data file size Sync time.Duration // Timeframe for syncing data Shrink time.Duration // Timeframe for shrinking data } diff --git a/db/db_test.go b/db/db_test.go index 1e3d2201..6765bdca 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -30,6 +30,7 @@ func setupDB() { cnf.Settings = &cnf.Options{} cnf.Settings.DB.Path = "memory" cnf.Settings.DB.Base = "*" + cnf.Settings.DB.Proc.Size = 5 workerCount = 1 @@ -93,6 +94,7 @@ func TestYield(t *testing.T) { res, err := Execute(setupKV(), txt, nil) So(err, ShouldBeNil) + So(res, ShouldHaveLength, 7) So(res[1].Result, ShouldHaveLength, 1) So(data.Consume(res[1].Result[0]).Get("test").Data(), ShouldEqual, 1) So(res[2].Result, ShouldHaveLength, 1) diff --git a/kvs/rixxdb/main.go b/kvs/rixxdb/main.go index dca15f94..cf862ce1 100644 --- a/kvs/rixxdb/main.go +++ b/kvs/rixxdb/main.go @@ -31,10 +31,16 @@ func init() { path := strings.TrimPrefix(opts.DB.Path, "rixxdb://") pntr, err = rixxdb.Open(path, &rixxdb.Config{ + // Set the encryption key + EncryptionKey: opts.DB.Key, + // Set the file size policy + SizePolicy: opts.DB.Proc.Size, + // Set the sync offset duration + SyncPolicy: opts.DB.Proc.Sync, + // Set the shrink offset duration + ShrinkPolicy: opts.DB.Proc.Shrink, + // Don't wait for syncing if shrinking IgnoreSyncPolicyWhenShrinking: true, - SyncPolicy: opts.DB.Proc.Sync, - ShrinkPolicy: opts.DB.Proc.Shrink, - EncryptionKey: opts.DB.Key, }) if err != nil {