add other session providers (#5963)

Gopkg.lock

@@ -17,6 +17,14 @@
   pruneopts = "NUT"
   revision = "d5a42771e7e851e8a89c5c6ffa0f5b075342f9df"
 
+[[projects]]
+  digest = "1:5d72bbcc9c8667b11c3dc3cbe681c5a6f71e5096744c0bf7726ab5c6425d5dc4"
+  name = "github.com/BurntSushi/toml"
+  packages = ["."]
+  pruneopts = "NUT"
+  revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005"
+  version = "v0.3.1"
+
 [[projects]]
   digest = "1:3fcef06a1a6561955c94af6c7757a6fa37605eb653f0d06ab960e5bb80092195"
   name = "github.com/PuerkitoBio/goquery"
@@ -185,6 +193,28 @@
   pruneopts = "NUT"
   revision = "098da33fde5f9220736531b3cb26a2dec86a8367"
 
+[[projects]]
+  branch = "master"
+  digest = "1:eb205556fe75307c6d2b58d4159e7c2da23e2666481d352c66d4055bebf45a3c"
+  name = "github.com/couchbase/gomemcached"
+  packages = [
+    ".",
+    "client",
+  ]
+  pruneopts = "NUT"
+  revision = "5125a94a666c83cb9b7a60907833cd320b84c20f"
+
+[[projects]]
+  branch = "master"
+  digest = "1:ea03e12e246f7708a7b7ab3ad04e96d21ce73f48bb56258bc2bffeed474212e6"
+  name = "github.com/couchbase/goutils"
+  packages = [
+    "logging",
+    "scramsha",
+  ]
+  pruneopts = "NUT"
+  revision = "e865a1461c8ac0032bd37e2d4dab3289faea3873"
+
 [[projects]]
   branch = "master"
   digest = "1:82e1ad11d777f7bff9a1fc678a8a534a318f85e5026a8a4d6f4a94a6b0678bb6"
@@ -197,6 +227,14 @@
   pruneopts = "NUT"
   revision = "eb6ae3743b3f300f2136f83ca78c08cc071edbd4"
 
+[[projects]]
+  branch = "master"
+  digest = "1:df592f4b82b993fcac270862376c34210776b8b0334a0f59f4d9d80467713ffa"
+  name = "github.com/couchbaselabs/go-couchbase"
+  packages = ["."]
+  pruneopts = "NUT"
+  revision = "d904413d884d1fb849e2ad8834619f661761ef57"
+
 [[projects]]
   digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39"
   name = "github.com/davecgh/go-spew"
@@ -358,14 +396,19 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:8fea5718d84af17762195beb6fe92a0d6c1048452a1dbc464d227f12e0cff0cc"
+  digest = "1:a26b7b56aece087165b8db87afb05db8495252449553ca20d15f5a24202f36bc"
   name = "github.com/go-macaron/session"
   packages = [
     ".",
+    "couchbase",
+    "memcache",
+    "mysql",
+    "nodb",
+    "postgres",
     "redis",
   ]
   pruneopts = "NUT"
-  revision = "330e4e4d8beb7b00111ac34539561f46f94c4458"
+  revision = "0a0a789bf1934e55fde19629869caa015a40c525"
 
 [[projects]]
   digest = "1:758d2371fcdee6d02565901b348729053c636055e67ef6e17aa466c7ff6cc57c"
@@ -579,6 +622,28 @@
   pruneopts = "NUT"
   revision = "e3534c89ef969912856dfa39e56b09e58c5f5daf"
 
+[[projects]]
+  digest = "1:1e6a29ed1f189354030e3371f63ec58aacbc2bf232fd104c6e0d41174ac5af48"
+  name = "github.com/lunny/log"
+  packages = ["."]
+  pruneopts = "NUT"
+  revision = "7887c61bf0de75586961948b286be6f7d05d9f58"
+  version = "v0.1"
+
+[[projects]]
+  branch = "master"
+  digest = "1:683d835728cb95d176d423b522420eb5e4ec859b276bca18466476b82b3ebc4c"
+  name = "github.com/lunny/nodb"
+  packages = [
+    ".",
+    "config",
+    "store",
+    "store/driver",
+    "store/goleveldb",
+  ]
+  pruneopts = "NUT"
+  revision = "fc1ef06ad4af0da31cdb87e3fa5ec084c67e6597"
+
 [[projects]]
   digest = "1:aa7dcd6a0db70d514821f8739d0a22e7df33b499d8d399cf15b2858d44f8319e"
   name = "github.com/markbates/goth"
@@ -682,6 +747,14 @@
   revision = "bb6d471dc95d4fe11e432687f8b70ff496cf3136"
   version = "v1.0.0"
 
+[[projects]]
+  digest = "1:14715f705ff5dfe0ffd6571d7d201dd8e921030f8070321a79380d8ca4ec1a24"
+  name = "github.com/pkg/errors"
+  packages = ["."]
+  pruneopts = "NUT"
+  revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
+  version = "v0.8.1"
+
 [[projects]]
   digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
   name = "github.com/pmezard/go-difflib"
@@ -775,6 +848,14 @@
   pruneopts = "NUT"
   revision = "1dba4b3954bc059efc3991ec364f9f9a35f597d2"
 
+[[projects]]
+  branch = "master"
+  digest = "1:dbda803f21e60c38de7d9f884390f2ebbe234ce0c3d139b65bbb36b03a99d266"
+  name = "github.com/siddontang/go-snappy"
+  packages = ["snappy"]
+  pruneopts = "NUT"
+  revision = "d8f7bb82a96d89c1254e5a6c967134e1433c9ee2"
+
 [[projects]]
   digest = "1:89fd77d603a74a6540d60067debad9397865bf040955d907362c95d364baeba6"
   name = "github.com/src-d/gcfg"
@@ -804,6 +885,27 @@
   revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
   version = "v1.2.1"
 
+[[projects]]
+  branch = "master"
+  digest = "1:685fdfea42d825ebd39ee0994354b46c374cf2c2b2d97a41a8dee1807c6a9b62"
+  name = "github.com/syndtr/goleveldb"
+  packages = [
+    "leveldb",
+    "leveldb/cache",
+    "leveldb/comparer",
+    "leveldb/errors",
+    "leveldb/filter",
+    "leveldb/iterator",
+    "leveldb/journal",
+    "leveldb/memdb",
+    "leveldb/opt",
+    "leveldb/storage",
+    "leveldb/table",
+    "leveldb/util",
+  ]
+  pruneopts = "NUT"
+  revision = "2f17a3356c6616cbfc4ae4c38147dc062a68fb0e"
+
 [[projects]]
   branch = "master"
   digest = "1:3cb6dfe7cdece5716b1c3c3c0b5faf7fce2e83e2758e2baad2e9986d101980b8"
@@ -1150,6 +1252,11 @@
     "github.com/go-macaron/i18n",
     "github.com/go-macaron/inject",
     "github.com/go-macaron/session",
+    "github.com/go-macaron/session/couchbase",
+    "github.com/go-macaron/session/memcache",
+    "github.com/go-macaron/session/mysql",
+    "github.com/go-macaron/session/nodb",
+    "github.com/go-macaron/session/postgres",
     "github.com/go-macaron/session/redis",
     "github.com/go-macaron/toolbox",
     "github.com/go-sql-driver/mysql",

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -250,7 +250,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 
 ## Session (`session`)
 
-- `PROVIDER`: **memory**: Session engine provider \[memory, file, redis, mysql\].
+- `PROVIDER`: **memory**: Session engine provider \[memory, file, redis, mysql, couchbase, memcache, nodb, postgres\].
 - `PROVIDER_CONFIG`: **data/sessions**: For file, the root path; for others, the connection string.
 - `COOKIE_SECURE`: **false**: Enable this to force using HTTPS for all session access.
 - `COOKIE_NAME`: **i\_like\_gitea**: The name of the cookie used for the session ID.

modules/setting/setting.go

@@ -31,7 +31,12 @@ import (
 	_ "github.com/go-macaron/cache/memcache" // memcache plugin for cache
 	_ "github.com/go-macaron/cache/redis"
 	"github.com/go-macaron/session"
-	_ "github.com/go-macaron/session/redis" // redis plugin for store session
+	_ "github.com/go-macaron/session/couchbase" // couchbase plugin for session store
+	_ "github.com/go-macaron/session/memcache"  // memcache plugin for session store
+	_ "github.com/go-macaron/session/mysql"     // mysql plugin for session store
+	_ "github.com/go-macaron/session/nodb"      // nodb plugin for session store
+	_ "github.com/go-macaron/session/postgres"  // postgres plugin for session store
+	_ "github.com/go-macaron/session/redis"     // redis plugin for store session
 	"github.com/go-xorm/core"
 	shellquote "github.com/kballard/go-shellquote"
 	version "github.com/mcuadros/go-version"
@@ -1506,7 +1511,7 @@ func newCacheService() {
 
 func newSessionService() {
 	SessionConfig.Provider = Cfg.Section("session").Key("PROVIDER").In("memory",
-		[]string{"memory", "file", "redis", "mysql"})
+		[]string{"memory", "file", "redis", "mysql", "postgres", "couchbase", "memcache", "nodb"})
 	SessionConfig.ProviderConfig = strings.Trim(Cfg.Section("session").Key("PROVIDER_CONFIG").MustString(path.Join(AppDataPath, "sessions")), "\" ")
 	if SessionConfig.Provider == "file" && !filepath.IsAbs(SessionConfig.ProviderConfig) {
 		SessionConfig.ProviderConfig = path.Join(AppWorkPath, SessionConfig.ProviderConfig)

vendor/github.com/BurntSushi/toml/COPYING

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 TOML authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 TOML authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 TOML authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 TOML authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/BurntSushi/toml/decode.go

@@ -0,0 +1,509 @@
+package toml
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math"
+	"reflect"
+	"strings"
+	"time"
+)
+
+func e(format string, args ...interface{}) error {
+	return fmt.Errorf("toml: "+format, args...)
+}
+
+// Unmarshaler is the interface implemented by objects that can unmarshal a
+// TOML description of themselves.
+type Unmarshaler interface {
+	UnmarshalTOML(interface{}) error
+}
+
+// Unmarshal decodes the contents of `p` in TOML format into a pointer `v`.
+func Unmarshal(p []byte, v interface{}) error {
+	_, err := Decode(string(p), v)
+	return err
+}
+
+// Primitive is a TOML value that hasn't been decoded into a Go value.
+// When using the various `Decode*` functions, the type `Primitive` may
+// be given to any value, and its decoding will be delayed.
+//
+// A `Primitive` value can be decoded using the `PrimitiveDecode` function.
+//
+// The underlying representation of a `Primitive` value is subject to change.
+// Do not rely on it.
+//
+// N.B. Primitive values are still parsed, so using them will only avoid
+// the overhead of reflection. They can be useful when you don't know the
+// exact type of TOML data until run time.
+type Primitive struct {
+	undecoded interface{}
+	context   Key
+}
+
+// DEPRECATED!
+//
+// Use MetaData.PrimitiveDecode instead.
+func PrimitiveDecode(primValue Primitive, v interface{}) error {
+	md := MetaData{decoded: make(map[string]bool)}
+	return md.unify(primValue.undecoded, rvalue(v))
+}
+
+// PrimitiveDecode is just like the other `Decode*` functions, except it
+// decodes a TOML value that has already been parsed. Valid primitive values
+// can *only* be obtained from values filled by the decoder functions,
+// including this method. (i.e., `v` may contain more `Primitive`
+// values.)
+//
+// Meta data for primitive values is included in the meta data returned by
+// the `Decode*` functions with one exception: keys returned by the Undecoded
+// method will only reflect keys that were decoded. Namely, any keys hidden
+// behind a Primitive will be considered undecoded. Executing this method will
+// update the undecoded keys in the meta data. (See the example.)
+func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error {
+	md.context = primValue.context
+	defer func() { md.context = nil }()
+	return md.unify(primValue.undecoded, rvalue(v))
+}
+
+// Decode will decode the contents of `data` in TOML format into a pointer
+// `v`.
+//
+// TOML hashes correspond to Go structs or maps. (Dealer's choice. They can be
+// used interchangeably.)
+//
+// TOML arrays of tables correspond to either a slice of structs or a slice
+// of maps.
+//
+// TOML datetimes correspond to Go `time.Time` values.
+//
+// All other TOML types (float, string, int, bool and array) correspond
+// to the obvious Go types.
+//
+// An exception to the above rules is if a type implements the
+// encoding.TextUnmarshaler interface. In this case, any primitive TOML value
+// (floats, strings, integers, booleans and datetimes) will be converted to
+// a byte string and given to the value's UnmarshalText method. See the
+// Unmarshaler example for a demonstration with time duration strings.
+//
+// Key mapping
+//
+// TOML keys can map to either keys in a Go map or field names in a Go
+// struct. The special `toml` struct tag may be used to map TOML keys to
+// struct fields that don't match the key name exactly. (See the example.)
+// A case insensitive match to struct names will be tried if an exact match
+// can't be found.
+//
+// The mapping between TOML values and Go values is loose. That is, there
+// may exist TOML values that cannot be placed into your representation, and
+// there may be parts of your representation that do not correspond to
+// TOML values. This loose mapping can be made stricter by using the IsDefined
+// and/or Undecoded methods on the MetaData returned.
+//
+// This decoder will not handle cyclic types. If a cyclic type is passed,
+// `Decode` will not terminate.
+func Decode(data string, v interface{}) (MetaData, error) {
+	rv := reflect.ValueOf(v)
+	if rv.Kind() != reflect.Ptr {
+		return MetaData{}, e("Decode of non-pointer %s", reflect.TypeOf(v))
+	}
+	if rv.IsNil() {
+		return MetaData{}, e("Decode of nil %s", reflect.TypeOf(v))
+	}
+	p, err := parse(data)
+	if err != nil {
+		return MetaData{}, err
+	}
+	md := MetaData{
+		p.mapping, p.types, p.ordered,
+		make(map[string]bool, len(p.ordered)), nil,
+	}
+	return md, md.unify(p.mapping, indirect(rv))
+}
+
+// DecodeFile is just like Decode, except it will automatically read the
+// contents of the file at `fpath` and decode it for you.
+func DecodeFile(fpath string, v interface{}) (MetaData, error) {
+	bs, err := ioutil.ReadFile(fpath)
+	if err != nil {
+		return MetaData{}, err
+	}
+	return Decode(string(bs), v)
+}
+
+// DecodeReader is just like Decode, except it will consume all bytes
+// from the reader and decode it for you.
+func DecodeReader(r io.Reader, v interface{}) (MetaData, error) {
+	bs, err := ioutil.ReadAll(r)
+	if err != nil {
+		return MetaData{}, err
+	}
+	return Decode(string(bs), v)
+}
+
+// unify performs a sort of type unification based on the structure of `rv`,
+// which is the client representation.
+//
+// Any type mismatch produces an error. Finding a type that we don't know
+// how to handle produces an unsupported type error.
+func (md *MetaData) unify(data interface{}, rv reflect.Value) error {
+
+	// Special case. Look for a `Primitive` value.
+	if rv.Type() == reflect.TypeOf((*Primitive)(nil)).Elem() {
+		// Save the undecoded data and the key context into the primitive
+		// value.
+		context := make(Key, len(md.context))
+		copy(context, md.context)
+		rv.Set(reflect.ValueOf(Primitive{
+			undecoded: data,
+			context:   context,
+		}))
+		return nil
+	}
+
+	// Special case. Unmarshaler Interface support.
+	if rv.CanAddr() {
+		if v, ok := rv.Addr().Interface().(Unmarshaler); ok {
+			return v.UnmarshalTOML(data)
+		}
+	}
+
+	// Special case. Handle time.Time values specifically.
+	// TODO: Remove this code when we decide to drop support for Go 1.1.
+	// This isn't necessary in Go 1.2 because time.Time satisfies the encoding
+	// interfaces.
+	if rv.Type().AssignableTo(rvalue(time.Time{}).Type()) {
+		return md.unifyDatetime(data, rv)
+	}
+
+	// Special case. Look for a value satisfying the TextUnmarshaler interface.
+	if v, ok := rv.Interface().(TextUnmarshaler); ok {
+		return md.unifyText(data, v)
+	}
+	// BUG(burntsushi)
+	// The behavior here is incorrect whenever a Go type satisfies the
+	// encoding.TextUnmarshaler interface but also corresponds to a TOML
+	// hash or array. In particular, the unmarshaler should only be applied
+	// to primitive TOML values. But at this point, it will be applied to
+	// all kinds of values and produce an incorrect error whenever those values
+	// are hashes or arrays (including arrays of tables).
+
+	k := rv.Kind()
+
+	// laziness
+	if k >= reflect.Int && k <= reflect.Uint64 {
+		return md.unifyInt(data, rv)
+	}
+	switch k {
+	case reflect.Ptr:
+		elem := reflect.New(rv.Type().Elem())
+		err := md.unify(data, reflect.Indirect(elem))
+		if err != nil {
+			return err
+		}
+		rv.Set(elem)
+		return nil
+	case reflect.Struct:
+		return md.unifyStruct(data, rv)
+	case reflect.Map:
+		return md.unifyMap(data, rv)
+	case reflect.Array:
+		return md.unifyArray(data, rv)
+	case reflect.Slice:
+		return md.unifySlice(data, rv)
+	case reflect.String:
+		return md.unifyString(data, rv)
+	case reflect.Bool:
+		return md.unifyBool(data, rv)
+	case reflect.Interface:
+		// we only support empty interfaces.
+		if rv.NumMethod() > 0 {
+			return e("unsupported type %s", rv.Type())
+		}
+		return md.unifyAnything(data, rv)
+	case reflect.Float32:
+		fallthrough
+	case reflect.Float64:
+		return md.unifyFloat64(data, rv)
+	}
+	return e("unsupported type %s", rv.Kind())
+}
+
+func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error {
+	tmap, ok := mapping.(map[string]interface{})
+	if !ok {
+		if mapping == nil {
+			return nil
+		}
+		return e("type mismatch for %s: expected table but found %T",
+			rv.Type().String(), mapping)
+	}
+
+	for key, datum := range tmap {
+		var f *field
+		fields := cachedTypeFields(rv.Type())
+		for i := range fields {
+			ff := &fields[i]
+			if ff.name == key {
+				f = ff
+				break
+			}
+			if f == nil && strings.EqualFold(ff.name, key) {
+				f = ff
+			}
+		}
+		if f != nil {
+			subv := rv
+			for _, i := range f.index {
+				subv = indirect(subv.Field(i))
+			}
+			if isUnifiable(subv) {
+				md.decoded[md.context.add(key).String()] = true
+				md.context = append(md.context, key)
+				if err := md.unify(datum, subv); err != nil {
+					return err
+				}
+				md.context = md.context[0 : len(md.context)-1]
+			} else if f.name != "" {
+				// Bad user! No soup for you!
+				return e("cannot write unexported field %s.%s",
+					rv.Type().String(), f.name)
+			}
+		}
+	}
+	return nil
+}
+
+func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error {
+	tmap, ok := mapping.(map[string]interface{})
+	if !ok {
+		if tmap == nil {
+			return nil
+		}
+		return badtype("map", mapping)
+	}
+	if rv.IsNil() {
+		rv.Set(reflect.MakeMap(rv.Type()))
+	}
+	for k, v := range tmap {
+		md.decoded[md.context.add(k).String()] = true
+		md.context = append(md.context, k)
+
+		rvkey := indirect(reflect.New(rv.Type().Key()))
+		rvval := reflect.Indirect(reflect.New(rv.Type().Elem()))
+		if err := md.unify(v, rvval); err != nil {
+			return err
+		}
+		md.context = md.context[0 : len(md.context)-1]
+
+		rvkey.SetString(k)
+		rv.SetMapIndex(rvkey, rvval)
+	}
+	return nil
+}
+
+func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error {
+	datav := reflect.ValueOf(data)
+	if datav.Kind() != reflect.Slice {
+		if !datav.IsValid() {
+			return nil
+		}
+		return badtype("slice", data)
+	}
+	sliceLen := datav.Len()
+	if sliceLen != rv.Len() {
+		return e("expected array length %d; got TOML array of length %d",
+			rv.Len(), sliceLen)
+	}
+	return md.unifySliceArray(datav, rv)
+}
+
+func (md *MetaData) unifySlice(data interface{}, rv reflect.Value) error {
+	datav := reflect.ValueOf(data)
+	if datav.Kind() != reflect.Slice {
+		if !datav.IsValid() {
+			return nil
+		}
+		return badtype("slice", data)
+	}
+	n := datav.Len()
+	if rv.IsNil() || rv.Cap() < n {
+		rv.Set(reflect.MakeSlice(rv.Type(), n, n))
+	}
+	rv.SetLen(n)
+	return md.unifySliceArray(datav, rv)
+}
+
+func (md *MetaData) unifySliceArray(data, rv reflect.Value) error {
+	sliceLen := data.Len()
+	for i := 0; i < sliceLen; i++ {
+		v := data.Index(i).Interface()
+		sliceval := indirect(rv.Index(i))
+		if err := md.unify(v, sliceval); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (md *MetaData) unifyDatetime(data interface{}, rv reflect.Value) error {
+	if _, ok := data.(time.Time); ok {
+		rv.Set(reflect.ValueOf(data))
+		return nil
+	}
+	return badtype("time.Time", data)
+}
+
+func (md *MetaData) unifyString(data interface{}, rv reflect.Value) error {
+	if s, ok := data.(string); ok {
+		rv.SetString(s)
+		return nil
+	}
+	return badtype("string", data)
+}
+
+func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error {
+	if num, ok := data.(float64); ok {
+		switch rv.Kind() {
+		case reflect.Float32:
+			fallthrough
+		case reflect.Float64:
+			rv.SetFloat(num)
+		default:
+			panic("bug")
+		}
+		return nil
+	}
+	return badtype("float", data)
+}
+
+func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error {
+	if num, ok := data.(int64); ok {
+		if rv.Kind() >= reflect.Int && rv.Kind() <= reflect.Int64 {
+			switch rv.Kind() {
+			case reflect.Int, reflect.Int64:
+				// No bounds checking necessary.
+			case reflect.Int8:
+				if num < math.MinInt8 || num > math.MaxInt8 {
+					return e("value %d is out of range for int8", num)
+				}
+			case reflect.Int16:
+				if num < math.MinInt16 || num > math.MaxInt16 {
+					return e("value %d is out of range for int16", num)
+				}
+			case reflect.Int32:
+				if num < math.MinInt32 || num > math.MaxInt32 {
+					return e("value %d is out of range for int32", num)
+				}
+			}
+			rv.SetInt(num)
+		} else if rv.Kind() >= reflect.Uint && rv.Kind() <= reflect.Uint64 {
+			unum := uint64(num)
+			switch rv.Kind() {
+			case reflect.Uint, reflect.Uint64:
+				// No bounds checking necessary.
+			case reflect.Uint8:
+				if num < 0 || unum > math.MaxUint8 {
+					return e("value %d is out of range for uint8", num)
+				}
+			case reflect.Uint16:
+				if num < 0 || unum > math.MaxUint16 {
+					return e("value %d is out of range for uint16", num)
+				}
+			case reflect.Uint32:
+				if num < 0 || unum > math.MaxUint32 {
+					return e("value %d is out of range for uint32", num)
+				}
+			}
+			rv.SetUint(unum)
+		} else {
+			panic("unreachable")
+		}
+		return nil
+	}
+	return badtype("integer", data)
+}
+
+func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error {
+	if b, ok := data.(bool); ok {
+		rv.SetBool(b)
+		return nil
+	}
+	return badtype("boolean", data)
+}
+
+func (md *MetaData) unifyAnything(data interface{}, rv reflect.Value) error {
+	rv.Set(reflect.ValueOf(data))
+	return nil
+}
+
+func (md *MetaData) unifyText(data interface{}, v TextUnmarshaler) error {
+	var s string
+	switch sdata := data.(type) {
+	case TextMarshaler:
+		text, err := sdata.MarshalText()
+		if err != nil {
+			return err
+		}
+		s = string(text)
+	case fmt.Stringer:
+		s = sdata.String()
+	case string:
+		s = sdata
+	case bool:
+		s = fmt.Sprintf("%v", sdata)
+	case int64:
+		s = fmt.Sprintf("%d", sdata)
+	case float64:
+		s = fmt.Sprintf("%f", sdata)
+	default:
+		return badtype("primitive (string-like)", data)
+	}
+	if err := v.UnmarshalText([]byte(s)); err != nil {
+		return err
+	}
+	return nil
+}
+
+// rvalue returns a reflect.Value of `v`. All pointers are resolved.
+func rvalue(v interface{}) reflect.Value {
+	return indirect(reflect.ValueOf(v))
+}
+
+// indirect returns the value pointed to by a pointer.
+// Pointers are followed until the value is not a pointer.
+// New values are allocated for each nil pointer.
+//
+// An exception to this rule is if the value satisfies an interface of
+// interest to us (like encoding.TextUnmarshaler).
+func indirect(v reflect.Value) reflect.Value {
+	if v.Kind() != reflect.Ptr {
+		if v.CanSet() {
+			pv := v.Addr()
+			if _, ok := pv.Interface().(TextUnmarshaler); ok {
+				return pv
+			}
+		}
+		return v
+	}
+	if v.IsNil() {
+		v.Set(reflect.New(v.Type().Elem()))
+	}
+	return indirect(reflect.Indirect(v))
+}
+
+func isUnifiable(rv reflect.Value) bool {
+	if rv.CanSet() {
+		return true
+	}
+	if _, ok := rv.Interface().(TextUnmarshaler); ok {
+		return true
+	}
+	return false
+}
+
+func badtype(expected string, data interface{}) error {
+	return e("cannot load TOML value of type %T into a Go %s", data, expected)
+}

vendor/github.com/BurntSushi/toml/decode_meta.go

@@ -0,0 +1,121 @@
+package toml
+
+import "strings"
+
+// MetaData allows access to meta information about TOML data that may not
+// be inferrable via reflection. In particular, whether a key has been defined
+// and the TOML type of a key.
+type MetaData struct {
+	mapping map[string]interface{}
+	types   map[string]tomlType
+	keys    []Key
+	decoded map[string]bool
+	context Key // Used only during decoding.
+}
+
+// IsDefined returns true if the key given exists in the TOML data. The key
+// should be specified hierarchially. e.g.,
+//
+//	// access the TOML key 'a.b.c'
+//	IsDefined("a", "b", "c")
+//
+// IsDefined will return false if an empty key given. Keys are case sensitive.
+func (md *MetaData) IsDefined(key ...string) bool {
+	if len(key) == 0 {
+		return false
+	}
+
+	var hash map[string]interface{}
+	var ok bool
+	var hashOrVal interface{} = md.mapping
+	for _, k := range key {
+		if hash, ok = hashOrVal.(map[string]interface{}); !ok {
+			return false
+		}
+		if hashOrVal, ok = hash[k]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+// Type returns a string representation of the type of the key specified.
+//
+// Type will return the empty string if given an empty key or a key that
+// does not exist. Keys are case sensitive.
+func (md *MetaData) Type(key ...string) string {
+	fullkey := strings.Join(key, ".")
+	if typ, ok := md.types[fullkey]; ok {
+		return typ.typeString()
+	}
+	return ""
+}
+
+// Key is the type of any TOML key, including key groups. Use (MetaData).Keys
+// to get values of this type.
+type Key []string
+
+func (k Key) String() string {
+	return strings.Join(k, ".")
+}
+
+func (k Key) maybeQuotedAll() string {
+	var ss []string
+	for i := range k {
+		ss = append(ss, k.maybeQuoted(i))
+	}
+	return strings.Join(ss, ".")
+}
+
+func (k Key) maybeQuoted(i int) string {
+	quote := false
+	for _, c := range k[i] {
+		if !isBareKeyChar(c) {
+			quote = true
+			break
+		}
+	}
+	if quote {
+		return "\"" + strings.Replace(k[i], "\"", "\\\"", -1) + "\""
+	}
+	return k[i]
+}
+
+func (k Key) add(piece string) Key {
+	newKey := make(Key, len(k)+1)
+	copy(newKey, k)
+	newKey[len(k)] = piece
+	return newKey
+}
+
+// Keys returns a slice of every key in the TOML data, including key groups.
+// Each key is itself a slice, where the first element is the top of the
+// hierarchy and the last is the most specific.
+//
+// The list will have the same order as the keys appeared in the TOML data.
+//
+// All keys returned are non-empty.
+func (md *MetaData) Keys() []Key {
+	return md.keys
+}
+
+// Undecoded returns all keys that have not been decoded in the order in which
+// they appear in the original TOML document.
+//
+// This includes keys that haven't been decoded because of a Primitive value.
+// Once the Primitive value is decoded, the keys will be considered decoded.
+//
+// Also note that decoding into an empty interface will result in no decoding,
+// and so no keys will be considered decoded.
+//
+// In this sense, the Undecoded keys correspond to keys in the TOML document
+// that do not have a concrete type in your representation.
+func (md *MetaData) Undecoded() []Key {
+	undecoded := make([]Key, 0, len(md.keys))
+	for _, key := range md.keys {
+		if !md.decoded[key.String()] {
+			undecoded = append(undecoded, key)
+		}
+	}
+	return undecoded
+}

vendor/github.com/BurntSushi/toml/doc.go

@@ -0,0 +1,27 @@
+/*
+Package toml provides facilities for decoding and encoding TOML configuration
+files via reflection. There is also support for delaying decoding with
+the Primitive type, and querying the set of keys in a TOML document with the
+MetaData type.
+
+The specification implemented: https://github.com/toml-lang/toml
+
+The sub-command github.com/BurntSushi/toml/cmd/tomlv can be used to verify
+whether a file is a valid TOML document. It can also be used to print the
+type of each key in a TOML document.
+
+Testing
+
+There are two important types of tests used for this package. The first is
+contained inside '*_test.go' files and uses the standard Go unit testing
+framework. These tests are primarily devoted to holistically testing the
+decoder and encoder.
+
+The second type of testing is used to verify the implementation's adherence
+to the TOML specification. These tests have been factored into their own
+project: https://github.com/BurntSushi/toml-test
+
+The reason the tests are in a separate project is so that they can be used by
+any implementation of TOML. Namely, it is language agnostic.
+*/
+package toml

vendor/github.com/BurntSushi/toml/encode.go

@@ -0,0 +1,568 @@
+package toml
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type tomlEncodeError struct{ error }
+
+var (
+	errArrayMixedElementTypes = errors.New(
+		"toml: cannot encode array with mixed element types")
+	errArrayNilElement = errors.New(
+		"toml: cannot encode array with nil element")
+	errNonString = errors.New(
+		"toml: cannot encode a map with non-string key type")
+	errAnonNonStruct = errors.New(
+		"toml: cannot encode an anonymous field that is not a struct")
+	errArrayNoTable = errors.New(
+		"toml: TOML array element cannot contain a table")
+	errNoKey = errors.New(
+		"toml: top-level values must be Go maps or structs")
+	errAnything = errors.New("") // used in testing
+)
+
+var quotedReplacer = strings.NewReplacer(
+	"\t", "\\t",
+	"\n", "\\n",
+	"\r", "\\r",
+	"\"", "\\\"",
+	"\\", "\\\\",
+)
+
+// Encoder controls the encoding of Go values to a TOML document to some
+// io.Writer.
+//
+// The indentation level can be controlled with the Indent field.
+type Encoder struct {
+	// A single indentation level. By default it is two spaces.
+	Indent string
+
+	// hasWritten is whether we have written any output to w yet.
+	hasWritten bool
+	w          *bufio.Writer
+}
+
+// NewEncoder returns a TOML encoder that encodes Go values to the io.Writer
+// given. By default, a single indentation level is 2 spaces.
+func NewEncoder(w io.Writer) *Encoder {
+	return &Encoder{
+		w:      bufio.NewWriter(w),
+		Indent: "  ",
+	}
+}
+
+// Encode writes a TOML representation of the Go value to the underlying
+// io.Writer. If the value given cannot be encoded to a valid TOML document,
+// then an error is returned.
+//
+// The mapping between Go values and TOML values should be precisely the same
+// as for the Decode* functions. Similarly, the TextMarshaler interface is
+// supported by encoding the resulting bytes as strings. (If you want to write
+// arbitrary binary data then you will need to use something like base64 since
+// TOML does not have any binary types.)
+//
+// When encoding TOML hashes (i.e., Go maps or structs), keys without any
+// sub-hashes are encoded first.
+//
+// If a Go map is encoded, then its keys are sorted alphabetically for
+// deterministic output. More control over this behavior may be provided if
+// there is demand for it.
+//
+// Encoding Go values without a corresponding TOML representation---like map
+// types with non-string keys---will cause an error to be returned. Similarly
+// for mixed arrays/slices, arrays/slices with nil elements, embedded
+// non-struct types and nested slices containing maps or structs.
+// (e.g., [][]map[string]string is not allowed but []map[string]string is OK
+// and so is []map[string][]string.)
+func (enc *Encoder) Encode(v interface{}) error {
+	rv := eindirect(reflect.ValueOf(v))
+	if err := enc.safeEncode(Key([]string{}), rv); err != nil {
+		return err
+	}
+	return enc.w.Flush()
+}
+
+func (enc *Encoder) safeEncode(key Key, rv reflect.Value) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			if terr, ok := r.(tomlEncodeError); ok {
+				err = terr.error
+				return
+			}
+			panic(r)
+		}
+	}()
+	enc.encode(key, rv)
+	return nil
+}
+
+func (enc *Encoder) encode(key Key, rv reflect.Value) {
+	// Special case. Time needs to be in ISO8601 format.
+	// Special case. If we can marshal the type to text, then we used that.
+	// Basically, this prevents the encoder for handling these types as
+	// generic structs (or whatever the underlying type of a TextMarshaler is).
+	switch rv.Interface().(type) {
+	case time.Time, TextMarshaler:
+		enc.keyEqElement(key, rv)
+		return
+	}
+
+	k := rv.Kind()
+	switch k {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
+		reflect.Int64,
+		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32,
+		reflect.Uint64,
+		reflect.Float32, reflect.Float64, reflect.String, reflect.Bool:
+		enc.keyEqElement(key, rv)
+	case reflect.Array, reflect.Slice:
+		if typeEqual(tomlArrayHash, tomlTypeOfGo(rv)) {
+			enc.eArrayOfTables(key, rv)
+		} else {
+			enc.keyEqElement(key, rv)
+		}
+	case reflect.Interface:
+		if rv.IsNil() {
+			return
+		}
+		enc.encode(key, rv.Elem())
+	case reflect.Map:
+		if rv.IsNil() {
+			return
+		}
+		enc.eTable(key, rv)
+	case reflect.Ptr:
+		if rv.IsNil() {
+			return
+		}
+		enc.encode(key, rv.Elem())
+	case reflect.Struct:
+		enc.eTable(key, rv)
+	default:
+		panic(e("unsupported type for key '%s': %s", key, k))
+	}
+}
+
+// eElement encodes any value that can be an array element (primitives and
+// arrays).
+func (enc *Encoder) eElement(rv reflect.Value) {
+	switch v := rv.Interface().(type) {
+	case time.Time:
+		// Special case time.Time as a primitive. Has to come before
+		// TextMarshaler below because time.Time implements
+		// encoding.TextMarshaler, but we need to always use UTC.
+		enc.wf(v.UTC().Format("2006-01-02T15:04:05Z"))
+		return
+	case TextMarshaler:
+		// Special case. Use text marshaler if it's available for this value.
+		if s, err := v.MarshalText(); err != nil {
+			encPanic(err)
+		} else {
+			enc.writeQuoted(string(s))
+		}
+		return
+	}
+	switch rv.Kind() {
+	case reflect.Bool:
+		enc.wf(strconv.FormatBool(rv.Bool()))
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
+		reflect.Int64:
+		enc.wf(strconv.FormatInt(rv.Int(), 10))
+	case reflect.Uint, reflect.Uint8, reflect.Uint16,
+		reflect.Uint32, reflect.Uint64:
+		enc.wf(strconv.FormatUint(rv.Uint(), 10))
+	case reflect.Float32:
+		enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 32)))
+	case reflect.Float64:
+		enc.wf(floatAddDecimal(strconv.FormatFloat(rv.Float(), 'f', -1, 64)))
+	case reflect.Array, reflect.Slice:
+		enc.eArrayOrSliceElement(rv)
+	case reflect.Interface:
+		enc.eElement(rv.Elem())
+	case reflect.String:
+		enc.writeQuoted(rv.String())
+	default:
+		panic(e("unexpected primitive type: %s", rv.Kind()))
+	}
+}
+
+// By the TOML spec, all floats must have a decimal with at least one
+// number on either side.
+func floatAddDecimal(fstr string) string {
+	if !strings.Contains(fstr, ".") {
+		return fstr + ".0"
+	}
+	return fstr
+}
+
+func (enc *Encoder) writeQuoted(s string) {
+	enc.wf("\"%s\"", quotedReplacer.Replace(s))
+}
+
+func (enc *Encoder) eArrayOrSliceElement(rv reflect.Value) {
+	length := rv.Len()
+	enc.wf("[")
+	for i := 0; i < length; i++ {
+		elem := rv.Index(i)
+		enc.eElement(elem)
+		if i != length-1 {
+			enc.wf(", ")
+		}
+	}
+	enc.wf("]")
+}
+
+func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) {
+	if len(key) == 0 {
+		encPanic(errNoKey)
+	}
+	for i := 0; i < rv.Len(); i++ {
+		trv := rv.Index(i)
+		if isNil(trv) {
+			continue
+		}
+		panicIfInvalidKey(key)
+		enc.newline()
+		enc.wf("%s[[%s]]", enc.indentStr(key), key.maybeQuotedAll())
+		enc.newline()
+		enc.eMapOrStruct(key, trv)
+	}
+}
+
+func (enc *Encoder) eTable(key Key, rv reflect.Value) {
+	panicIfInvalidKey(key)
+	if len(key) == 1 {
+		// Output an extra newline between top-level tables.
+		// (The newline isn't written if nothing else has been written though.)
+		enc.newline()
+	}
+	if len(key) > 0 {
+		enc.wf("%s[%s]", enc.indentStr(key), key.maybeQuotedAll())
+		enc.newline()
+	}
+	enc.eMapOrStruct(key, rv)
+}
+
+func (enc *Encoder) eMapOrStruct(key Key, rv reflect.Value) {
+	switch rv := eindirect(rv); rv.Kind() {
+	case reflect.Map:
+		enc.eMap(key, rv)
+	case reflect.Struct:
+		enc.eStruct(key, rv)
+	default:
+		panic("eTable: unhandled reflect.Value Kind: " + rv.Kind().String())
+	}
+}
+
+func (enc *Encoder) eMap(key Key, rv reflect.Value) {
+	rt := rv.Type()
+	if rt.Key().Kind() != reflect.String {
+		encPanic(errNonString)
+	}
+
+	// Sort keys so that we have deterministic output. And write keys directly
+	// underneath this key first, before writing sub-structs or sub-maps.
+	var mapKeysDirect, mapKeysSub []string
+	for _, mapKey := range rv.MapKeys() {
+		k := mapKey.String()
+		if typeIsHash(tomlTypeOfGo(rv.MapIndex(mapKey))) {
+			mapKeysSub = append(mapKeysSub, k)
+		} else {
+			mapKeysDirect = append(mapKeysDirect, k)
+		}
+	}
+
+	var writeMapKeys = func(mapKeys []string) {
+		sort.Strings(mapKeys)
+		for _, mapKey := range mapKeys {
+			mrv := rv.MapIndex(reflect.ValueOf(mapKey))
+			if isNil(mrv) {
+				// Don't write anything for nil fields.
+				continue
+			}
+			enc.encode(key.add(mapKey), mrv)
+		}
+	}
+	writeMapKeys(mapKeysDirect)
+	writeMapKeys(mapKeysSub)
+}
+
+func (enc *Encoder) eStruct(key Key, rv reflect.Value) {
+	// Write keys for fields directly under this key first, because if we write
+	// a field that creates a new table, then all keys under it will be in that
+	// table (not the one we're writing here).
+	rt := rv.Type()
+	var fieldsDirect, fieldsSub [][]int
+	var addFields func(rt reflect.Type, rv reflect.Value, start []int)
+	addFields = func(rt reflect.Type, rv reflect.Value, start []int) {
+		for i := 0; i < rt.NumField(); i++ {
+			f := rt.Field(i)
+			// skip unexported fields
+			if f.PkgPath != "" && !f.Anonymous {
+				continue
+			}
+			frv := rv.Field(i)
+			if f.Anonymous {
+				t := f.Type
+				switch t.Kind() {
+				case reflect.Struct:
+					// Treat anonymous struct fields with
+					// tag names as though they are not
+					// anonymous, like encoding/json does.
+					if getOptions(f.Tag).name == "" {
+						addFields(t, frv, f.Index)
+						continue
+					}
+				case reflect.Ptr:
+					if t.Elem().Kind() == reflect.Struct &&
+						getOptions(f.Tag).name == "" {
+						if !frv.IsNil() {
+							addFields(t.Elem(), frv.Elem(), f.Index)
+						}
+						continue
+					}
+					// Fall through to the normal field encoding logic below
+					// for non-struct anonymous fields.
+				}
+			}
+
+			if typeIsHash(tomlTypeOfGo(frv)) {
+				fieldsSub = append(fieldsSub, append(start, f.Index...))
+			} else {
+				fieldsDirect = append(fieldsDirect, append(start, f.Index...))
+			}
+		}
+	}
+	addFields(rt, rv, nil)
+
+	var writeFields = func(fields [][]int) {
+		for _, fieldIndex := range fields {
+			sft := rt.FieldByIndex(fieldIndex)
+			sf := rv.FieldByIndex(fieldIndex)
+			if isNil(sf) {
+				// Don't write anything for nil fields.
+				continue
+			}
+
+			opts := getOptions(sft.Tag)
+			if opts.skip {
+				continue
+			}
+			keyName := sft.Name
+			if opts.name != "" {
+				keyName = opts.name
+			}
+			if opts.omitempty && isEmpty(sf) {
+				continue
+			}
+			if opts.omitzero && isZero(sf) {
+				continue
+			}
+
+			enc.encode(key.add(keyName), sf)
+		}
+	}
+	writeFields(fieldsDirect)
+	writeFields(fieldsSub)
+}
+
+// tomlTypeName returns the TOML type name of the Go value's type. It is
+// used to determine whether the types of array elements are mixed (which is
+// forbidden). If the Go value is nil, then it is illegal for it to be an array
+// element, and valueIsNil is returned as true.
+
+// Returns the TOML type of a Go value. The type may be `nil`, which means
+// no concrete TOML type could be found.
+func tomlTypeOfGo(rv reflect.Value) tomlType {
+	if isNil(rv) || !rv.IsValid() {
+		return nil
+	}
+	switch rv.Kind() {
+	case reflect.Bool:
+		return tomlBool
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32,
+		reflect.Int64,
+		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32,
+		reflect.Uint64:
+		return tomlInteger
+	case reflect.Float32, reflect.Float64:
+		return tomlFloat
+	case reflect.Array, reflect.Slice:
+		if typeEqual(tomlHash, tomlArrayType(rv)) {
+			return tomlArrayHash
+		}
+		return tomlArray
+	case reflect.Ptr, reflect.Interface:
+		return tomlTypeOfGo(rv.Elem())
+	case reflect.String:
+		return tomlString
+	case reflect.Map:
+		return tomlHash
+	case reflect.Struct:
+		switch rv.Interface().(type) {
+		case time.Time:
+			return tomlDatetime
+		case TextMarshaler:
+			return tomlString
+		default:
+			return tomlHash
+		}
+	default:
+		panic("unexpected reflect.Kind: " + rv.Kind().String())
+	}
+}
+
+// tomlArrayType returns the element type of a TOML array. The type returned
+// may be nil if it cannot be determined (e.g., a nil slice or a zero length
+// slize). This function may also panic if it finds a type that cannot be
+// expressed in TOML (such as nil elements, heterogeneous arrays or directly
+// nested arrays of tables).
+func tomlArrayType(rv reflect.Value) tomlType {
+	if isNil(rv) || !rv.IsValid() || rv.Len() == 0 {
+		return nil
+	}
+	firstType := tomlTypeOfGo(rv.Index(0))
+	if firstType == nil {
+		encPanic(errArrayNilElement)
+	}
+
+	rvlen := rv.Len()
+	for i := 1; i < rvlen; i++ {
+		elem := rv.Index(i)
+		switch elemType := tomlTypeOfGo(elem); {
+		case elemType == nil:
+			encPanic(errArrayNilElement)
+		case !typeEqual(firstType, elemType):
+			encPanic(errArrayMixedElementTypes)
+		}
+	}
+	// If we have a nested array, then we must make sure that the nested
+	// array contains ONLY primitives.
+	// This checks arbitrarily nested arrays.
+	if typeEqual(firstType, tomlArray) || typeEqual(firstType, tomlArrayHash) {
+		nest := tomlArrayType(eindirect(rv.Index(0)))
+		if typeEqual(nest, tomlHash) || typeEqual(nest, tomlArrayHash) {
+			encPanic(errArrayNoTable)
+		}
+	}
+	return firstType
+}
+
+type tagOptions struct {
+	skip      bool // "-"
+	name      string
+	omitempty bool
+	omitzero  bool
+}
+
+func getOptions(tag reflect.StructTag) tagOptions {
+	t := tag.Get("toml")
+	if t == "-" {
+		return tagOptions{skip: true}
+	}
+	var opts tagOptions
+	parts := strings.Split(t, ",")
+	opts.name = parts[0]
+	for _, s := range parts[1:] {
+		switch s {
+		case "omitempty":
+			opts.omitempty = true
+		case "omitzero":
+			opts.omitzero = true
+		}
+	}
+	return opts
+}
+
+func isZero(rv reflect.Value) bool {
+	switch rv.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return rv.Int() == 0
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return rv.Uint() == 0
+	case reflect.Float32, reflect.Float64:
+		return rv.Float() == 0.0
+	}
+	return false
+}
+
+func isEmpty(rv reflect.Value) bool {
+	switch rv.Kind() {
+	case reflect.Array, reflect.Slice, reflect.Map, reflect.String:
+		return rv.Len() == 0
+	case reflect.Bool:
+		return !rv.Bool()
+	}
+	return false
+}
+
+func (enc *Encoder) newline() {
+	if enc.hasWritten {
+		enc.wf("\n")
+	}
+}
+
+func (enc *Encoder) keyEqElement(key Key, val reflect.Value) {
+	if len(key) == 0 {
+		encPanic(errNoKey)
+	}
+	panicIfInvalidKey(key)
+	enc.wf("%s%s = ", enc.indentStr(key), key.maybeQuoted(len(key)-1))
+	enc.eElement(val)
+	enc.newline()
+}
+
+func (enc *Encoder) wf(format string, v ...interface{}) {
+	if _, err := fmt.Fprintf(enc.w, format, v...); err != nil {
+		encPanic(err)
+	}
+	enc.hasWritten = true
+}
+
+func (enc *Encoder) indentStr(key Key) string {
+	return strings.Repeat(enc.Indent, len(key)-1)
+}
+
+func encPanic(err error) {
+	panic(tomlEncodeError{err})
+}
+
+func eindirect(v reflect.Value) reflect.Value {
+	switch v.Kind() {
+	case reflect.Ptr, reflect.Interface:
+		return eindirect(v.Elem())
+	default:
+		return v
+	}
+}
+
+func isNil(rv reflect.Value) bool {
+	switch rv.Kind() {
+	case reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
+		return rv.IsNil()
+	default:
+		return false
+	}
+}
+
+func panicIfInvalidKey(key Key) {
+	for _, k := range key {
+		if len(k) == 0 {
+			encPanic(e("Key '%s' is not a valid table name. Key names "+
+				"cannot be empty.", key.maybeQuotedAll()))
+		}
+	}
+}
+
+func isValidKeyName(s string) bool {
+	return len(s) != 0
+}

vendor/github.com/BurntSushi/toml/encoding_types.go

@@ -0,0 +1,19 @@
+// +build go1.2
+
+package toml
+
+// In order to support Go 1.1, we define our own TextMarshaler and
+// TextUnmarshaler types. For Go 1.2+, we just alias them with the
+// standard library interfaces.
+
+import (
+	"encoding"
+)
+
+// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here
+// so that Go 1.1 can be supported.
+type TextMarshaler encoding.TextMarshaler
+
+// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined
+// here so that Go 1.1 can be supported.
+type TextUnmarshaler encoding.TextUnmarshaler

vendor/github.com/BurntSushi/toml/encoding_types_1.1.go

@@ -0,0 +1,18 @@
+// +build !go1.2
+
+package toml
+
+// These interfaces were introduced in Go 1.2, so we add them manually when
+// compiling for Go 1.1.
+
+// TextMarshaler is a synonym for encoding.TextMarshaler. It is defined here
+// so that Go 1.1 can be supported.
+type TextMarshaler interface {
+	MarshalText() (text []byte, err error)
+}
+
+// TextUnmarshaler is a synonym for encoding.TextUnmarshaler. It is defined
+// here so that Go 1.1 can be supported.
+type TextUnmarshaler interface {
+	UnmarshalText(text []byte) error
+}

vendor/github.com/BurntSushi/toml/lex.go

@@ -0,0 +1,953 @@
+package toml
+
+import (
+	"fmt"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+type itemType int
+
+const (
+	itemError itemType = iota
+	itemNIL            // used in the parser to indicate no type
+	itemEOF
+	itemText
+	itemString
+	itemRawString
+	itemMultilineString
+	itemRawMultilineString
+	itemBool
+	itemInteger
+	itemFloat
+	itemDatetime
+	itemArray // the start of an array
+	itemArrayEnd
+	itemTableStart
+	itemTableEnd
+	itemArrayTableStart
+	itemArrayTableEnd
+	itemKeyStart
+	itemCommentStart
+	itemInlineTableStart
+	itemInlineTableEnd
+)
+
+const (
+	eof              = 0
+	comma            = ','
+	tableStart       = '['
+	tableEnd         = ']'
+	arrayTableStart  = '['
+	arrayTableEnd    = ']'
+	tableSep         = '.'
+	keySep           = '='
+	arrayStart       = '['
+	arrayEnd         = ']'
+	commentStart     = '#'
+	stringStart      = '"'
+	stringEnd        = '"'
+	rawStringStart   = '\''
+	rawStringEnd     = '\''
+	inlineTableStart = '{'
+	inlineTableEnd   = '}'
+)
+
+type stateFn func(lx *lexer) stateFn
+
+type lexer struct {
+	input string
+	start int
+	pos   int
+	line  int
+	state stateFn
+	items chan item
+
+	// Allow for backing up up to three runes.
+	// This is necessary because TOML contains 3-rune tokens (""" and ''').
+	prevWidths [3]int
+	nprev      int // how many of prevWidths are in use
+	// If we emit an eof, we can still back up, but it is not OK to call
+	// next again.
+	atEOF bool
+
+	// A stack of state functions used to maintain context.
+	// The idea is to reuse parts of the state machine in various places.
+	// For example, values can appear at the top level or within arbitrarily
+	// nested arrays. The last state on the stack is used after a value has
+	// been lexed. Similarly for comments.
+	stack []stateFn
+}
+
+type item struct {
+	typ  itemType
+	val  string
+	line int
+}
+
+func (lx *lexer) nextItem() item {
+	for {
+		select {
+		case item := <-lx.items:
+			return item
+		default:
+			lx.state = lx.state(lx)
+		}
+	}
+}
+
+func lex(input string) *lexer {
+	lx := &lexer{
+		input: input,
+		state: lexTop,
+		line:  1,
+		items: make(chan item, 10),
+		stack: make([]stateFn, 0, 10),
+	}
+	return lx
+}
+
+func (lx *lexer) push(state stateFn) {
+	lx.stack = append(lx.stack, state)
+}
+
+func (lx *lexer) pop() stateFn {
+	if len(lx.stack) == 0 {
+		return lx.errorf("BUG in lexer: no states to pop")
+	}
+	last := lx.stack[len(lx.stack)-1]
+	lx.stack = lx.stack[0 : len(lx.stack)-1]
+	return last
+}
+
+func (lx *lexer) current() string {
+	return lx.input[lx.start:lx.pos]
+}
+
+func (lx *lexer) emit(typ itemType) {
+	lx.items <- item{typ, lx.current(), lx.line}
+	lx.start = lx.pos
+}
+
+func (lx *lexer) emitTrim(typ itemType) {
+	lx.items <- item{typ, strings.TrimSpace(lx.current()), lx.line}
+	lx.start = lx.pos
+}
+
+func (lx *lexer) next() (r rune) {
+	if lx.atEOF {
+		panic("next called after EOF")
+	}
+	if lx.pos >= len(lx.input) {
+		lx.atEOF = true
+		return eof
+	}
+
+	if lx.input[lx.pos] == '\n' {
+		lx.line++
+	}
+	lx.prevWidths[2] = lx.prevWidths[1]
+	lx.prevWidths[1] = lx.prevWidths[0]
+	if lx.nprev < 3 {
+		lx.nprev++
+	}
+	r, w := utf8.DecodeRuneInString(lx.input[lx.pos:])
+	lx.prevWidths[0] = w
+	lx.pos += w
+	return r
+}
+
+// ignore skips over the pending input before this point.
+func (lx *lexer) ignore() {
+	lx.start = lx.pos
+}
+
+// backup steps back one rune. Can be called only twice between calls to next.
+func (lx *lexer) backup() {
+	if lx.atEOF {
+		lx.atEOF = false
+		return
+	}
+	if lx.nprev < 1 {
+		panic("backed up too far")
+	}
+	w := lx.prevWidths[0]
+	lx.prevWidths[0] = lx.prevWidths[1]
+	lx.prevWidths[1] = lx.prevWidths[2]
+	lx.nprev--
+	lx.pos -= w
+	if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' {
+		lx.line--
+	}
+}
+
+// accept consumes the next rune if it's equal to `valid`.
+func (lx *lexer) accept(valid rune) bool {
+	if lx.next() == valid {
+		return true
+	}
+	lx.backup()
+	return false
+}
+
+// peek returns but does not consume the next rune in the input.
+func (lx *lexer) peek() rune {
+	r := lx.next()
+	lx.backup()
+	return r
+}
+
+// skip ignores all input that matches the given predicate.
+func (lx *lexer) skip(pred func(rune) bool) {
+	for {
+		r := lx.next()
+		if pred(r) {
+			continue
+		}
+		lx.backup()
+		lx.ignore()
+		return
+	}
+}
+
+// errorf stops all lexing by emitting an error and returning `nil`.
+// Note that any value that is a character is escaped if it's a special
+// character (newlines, tabs, etc.).
+func (lx *lexer) errorf(format string, values ...interface{}) stateFn {
+	lx.items <- item{
+		itemError,
+		fmt.Sprintf(format, values...),
+		lx.line,
+	}
+	return nil
+}
+
+// lexTop consumes elements at the top level of TOML data.
+func lexTop(lx *lexer) stateFn {
+	r := lx.next()
+	if isWhitespace(r) || isNL(r) {
+		return lexSkip(lx, lexTop)
+	}
+	switch r {
+	case commentStart:
+		lx.push(lexTop)
+		return lexCommentStart
+	case tableStart:
+		return lexTableStart
+	case eof:
+		if lx.pos > lx.start {
+			return lx.errorf("unexpected EOF")
+		}
+		lx.emit(itemEOF)
+		return nil
+	}
+
+	// At this point, the only valid item can be a key, so we back up
+	// and let the key lexer do the rest.
+	lx.backup()
+	lx.push(lexTopEnd)
+	return lexKeyStart
+}
+
+// lexTopEnd is entered whenever a top-level item has been consumed. (A value
+// or a table.) It must see only whitespace, and will turn back to lexTop
+// upon a newline. If it sees EOF, it will quit the lexer successfully.
+func lexTopEnd(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case r == commentStart:
+		// a comment will read to a newline for us.
+		lx.push(lexTop)
+		return lexCommentStart
+	case isWhitespace(r):
+		return lexTopEnd
+	case isNL(r):
+		lx.ignore()
+		return lexTop
+	case r == eof:
+		lx.emit(itemEOF)
+		return nil
+	}
+	return lx.errorf("expected a top-level item to end with a newline, "+
+		"comment, or EOF, but got %q instead", r)
+}
+
+// lexTable lexes the beginning of a table. Namely, it makes sure that
+// it starts with a character other than '.' and ']'.
+// It assumes that '[' has already been consumed.
+// It also handles the case that this is an item in an array of tables.
+// e.g., '[[name]]'.
+func lexTableStart(lx *lexer) stateFn {
+	if lx.peek() == arrayTableStart {
+		lx.next()
+		lx.emit(itemArrayTableStart)
+		lx.push(lexArrayTableEnd)
+	} else {
+		lx.emit(itemTableStart)
+		lx.push(lexTableEnd)
+	}
+	return lexTableNameStart
+}
+
+func lexTableEnd(lx *lexer) stateFn {
+	lx.emit(itemTableEnd)
+	return lexTopEnd
+}
+
+func lexArrayTableEnd(lx *lexer) stateFn {
+	if r := lx.next(); r != arrayTableEnd {
+		return lx.errorf("expected end of table array name delimiter %q, "+
+			"but got %q instead", arrayTableEnd, r)
+	}
+	lx.emit(itemArrayTableEnd)
+	return lexTopEnd
+}
+
+func lexTableNameStart(lx *lexer) stateFn {
+	lx.skip(isWhitespace)
+	switch r := lx.peek(); {
+	case r == tableEnd || r == eof:
+		return lx.errorf("unexpected end of table name " +
+			"(table names cannot be empty)")
+	case r == tableSep:
+		return lx.errorf("unexpected table separator " +
+			"(table names cannot be empty)")
+	case r == stringStart || r == rawStringStart:
+		lx.ignore()
+		lx.push(lexTableNameEnd)
+		return lexValue // reuse string lexing
+	default:
+		return lexBareTableName
+	}
+}
+
+// lexBareTableName lexes the name of a table. It assumes that at least one
+// valid character for the table has already been read.
+func lexBareTableName(lx *lexer) stateFn {
+	r := lx.next()
+	if isBareKeyChar(r) {
+		return lexBareTableName
+	}
+	lx.backup()
+	lx.emit(itemText)
+	return lexTableNameEnd
+}
+
+// lexTableNameEnd reads the end of a piece of a table name, optionally
+// consuming whitespace.
+func lexTableNameEnd(lx *lexer) stateFn {
+	lx.skip(isWhitespace)
+	switch r := lx.next(); {
+	case isWhitespace(r):
+		return lexTableNameEnd
+	case r == tableSep:
+		lx.ignore()
+		return lexTableNameStart
+	case r == tableEnd:
+		return lx.pop()
+	default:
+		return lx.errorf("expected '.' or ']' to end table name, "+
+			"but got %q instead", r)
+	}
+}
+
+// lexKeyStart consumes a key name up until the first non-whitespace character.
+// lexKeyStart will ignore whitespace.
+func lexKeyStart(lx *lexer) stateFn {
+	r := lx.peek()
+	switch {
+	case r == keySep:
+		return lx.errorf("unexpected key separator %q", keySep)
+	case isWhitespace(r) || isNL(r):
+		lx.next()
+		return lexSkip(lx, lexKeyStart)
+	case r == stringStart || r == rawStringStart:
+		lx.ignore()
+		lx.emit(itemKeyStart)
+		lx.push(lexKeyEnd)
+		return lexValue // reuse string lexing
+	default:
+		lx.ignore()
+		lx.emit(itemKeyStart)
+		return lexBareKey
+	}
+}
+
+// lexBareKey consumes the text of a bare key. Assumes that the first character
+// (which is not whitespace) has not yet been consumed.
+func lexBareKey(lx *lexer) stateFn {
+	switch r := lx.next(); {
+	case isBareKeyChar(r):
+		return lexBareKey
+	case isWhitespace(r):
+		lx.backup()
+		lx.emit(itemText)
+		return lexKeyEnd
+	case r == keySep:
+		lx.backup()
+		lx.emit(itemText)
+		return lexKeyEnd
+	default:
+		return lx.errorf("bare keys cannot contain %q", r)
+	}
+}
+
+// lexKeyEnd consumes the end of a key and trims whitespace (up to the key
+// separator).
+func lexKeyEnd(lx *lexer) stateFn {
+	switch r := lx.next(); {
+	case r == keySep:
+		return lexSkip(lx, lexValue)
+	case isWhitespace(r):
+		return lexSkip(lx, lexKeyEnd)
+	default:
+		return lx.errorf("expected key separator %q, but got %q instead",
+			keySep, r)
+	}
+}
+
+// lexValue starts the consumption of a value anywhere a value is expected.
+// lexValue will ignore whitespace.
+// After a value is lexed, the last state on the next is popped and returned.
+func lexValue(lx *lexer) stateFn {
+	// We allow whitespace to precede a value, but NOT newlines.
+	// In array syntax, the array states are responsible for ignoring newlines.
+	r := lx.next()
+	switch {
+	case isWhitespace(r):
+		return lexSkip(lx, lexValue)
+	case isDigit(r):
+		lx.backup() // avoid an extra state and use the same as above
+		return lexNumberOrDateStart
+	}
+	switch r {
+	case arrayStart:
+		lx.ignore()
+		lx.emit(itemArray)
+		return lexArrayValue
+	case inlineTableStart:
+		lx.ignore()
+		lx.emit(itemInlineTableStart)
+		return lexInlineTableValue
+	case stringStart:
+		if lx.accept(stringStart) {
+			if lx.accept(stringStart) {
+				lx.ignore() // Ignore """
+				return lexMultilineString
+			}
+			lx.backup()
+		}
+		lx.ignore() // ignore the '"'
+		return lexString
+	case rawStringStart:
+		if lx.accept(rawStringStart) {
+			if lx.accept(rawStringStart) {
+				lx.ignore() // Ignore """
+				return lexMultilineRawString
+			}
+			lx.backup()
+		}
+		lx.ignore() // ignore the "'"
+		return lexRawString
+	case '+', '-':
+		return lexNumberStart
+	case '.': // special error case, be kind to users
+		return lx.errorf("floats must start with a digit, not '.'")
+	}
+	if unicode.IsLetter(r) {
+		// Be permissive here; lexBool will give a nice error if the
+		// user wrote something like
+		//   x = foo
+		// (i.e. not 'true' or 'false' but is something else word-like.)
+		lx.backup()
+		return lexBool
+	}
+	return lx.errorf("expected value but found %q instead", r)
+}
+
+// lexArrayValue consumes one value in an array. It assumes that '[' or ','
+// have already been consumed. All whitespace and newlines are ignored.
+func lexArrayValue(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case isWhitespace(r) || isNL(r):
+		return lexSkip(lx, lexArrayValue)
+	case r == commentStart:
+		lx.push(lexArrayValue)
+		return lexCommentStart
+	case r == comma:
+		return lx.errorf("unexpected comma")
+	case r == arrayEnd:
+		// NOTE(caleb): The spec isn't clear about whether you can have
+		// a trailing comma or not, so we'll allow it.
+		return lexArrayEnd
+	}
+
+	lx.backup()
+	lx.push(lexArrayValueEnd)
+	return lexValue
+}
+
+// lexArrayValueEnd consumes everything between the end of an array value and
+// the next value (or the end of the array): it ignores whitespace and newlines
+// and expects either a ',' or a ']'.
+func lexArrayValueEnd(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case isWhitespace(r) || isNL(r):
+		return lexSkip(lx, lexArrayValueEnd)
+	case r == commentStart:
+		lx.push(lexArrayValueEnd)
+		return lexCommentStart
+	case r == comma:
+		lx.ignore()
+		return lexArrayValue // move on to the next value
+	case r == arrayEnd:
+		return lexArrayEnd
+	}
+	return lx.errorf(
+		"expected a comma or array terminator %q, but got %q instead",
+		arrayEnd, r,
+	)
+}
+
+// lexArrayEnd finishes the lexing of an array.
+// It assumes that a ']' has just been consumed.
+func lexArrayEnd(lx *lexer) stateFn {
+	lx.ignore()
+	lx.emit(itemArrayEnd)
+	return lx.pop()
+}
+
+// lexInlineTableValue consumes one key/value pair in an inline table.
+// It assumes that '{' or ',' have already been consumed. Whitespace is ignored.
+func lexInlineTableValue(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case isWhitespace(r):
+		return lexSkip(lx, lexInlineTableValue)
+	case isNL(r):
+		return lx.errorf("newlines not allowed within inline tables")
+	case r == commentStart:
+		lx.push(lexInlineTableValue)
+		return lexCommentStart
+	case r == comma:
+		return lx.errorf("unexpected comma")
+	case r == inlineTableEnd:
+		return lexInlineTableEnd
+	}
+	lx.backup()
+	lx.push(lexInlineTableValueEnd)
+	return lexKeyStart
+}
+
+// lexInlineTableValueEnd consumes everything between the end of an inline table
+// key/value pair and the next pair (or the end of the table):
+// it ignores whitespace and expects either a ',' or a '}'.
+func lexInlineTableValueEnd(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case isWhitespace(r):
+		return lexSkip(lx, lexInlineTableValueEnd)
+	case isNL(r):
+		return lx.errorf("newlines not allowed within inline tables")
+	case r == commentStart:
+		lx.push(lexInlineTableValueEnd)
+		return lexCommentStart
+	case r == comma:
+		lx.ignore()
+		return lexInlineTableValue
+	case r == inlineTableEnd:
+		return lexInlineTableEnd
+	}
+	return lx.errorf("expected a comma or an inline table terminator %q, "+
+		"but got %q instead", inlineTableEnd, r)
+}
+
+// lexInlineTableEnd finishes the lexing of an inline table.
+// It assumes that a '}' has just been consumed.
+func lexInlineTableEnd(lx *lexer) stateFn {
+	lx.ignore()
+	lx.emit(itemInlineTableEnd)
+	return lx.pop()
+}
+
+// lexString consumes the inner contents of a string. It assumes that the
+// beginning '"' has already been consumed and ignored.
+func lexString(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case r == eof:
+		return lx.errorf("unexpected EOF")
+	case isNL(r):
+		return lx.errorf("strings cannot contain newlines")
+	case r == '\\':
+		lx.push(lexString)
+		return lexStringEscape
+	case r == stringEnd:
+		lx.backup()
+		lx.emit(itemString)
+		lx.next()
+		lx.ignore()
+		return lx.pop()
+	}
+	return lexString
+}
+
+// lexMultilineString consumes the inner contents of a string. It assumes that
+// the beginning '"""' has already been consumed and ignored.
+func lexMultilineString(lx *lexer) stateFn {
+	switch lx.next() {
+	case eof:
+		return lx.errorf("unexpected EOF")
+	case '\\':
+		return lexMultilineStringEscape
+	case stringEnd:
+		if lx.accept(stringEnd) {
+			if lx.accept(stringEnd) {
+				lx.backup()
+				lx.backup()
+				lx.backup()
+				lx.emit(itemMultilineString)
+				lx.next()
+				lx.next()
+				lx.next()
+				lx.ignore()
+				return lx.pop()
+			}
+			lx.backup()
+		}
+	}
+	return lexMultilineString
+}
+
+// lexRawString consumes a raw string. Nothing can be escaped in such a string.
+// It assumes that the beginning "'" has already been consumed and ignored.
+func lexRawString(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case r == eof:
+		return lx.errorf("unexpected EOF")
+	case isNL(r):
+		return lx.errorf("strings cannot contain newlines")
+	case r == rawStringEnd:
+		lx.backup()
+		lx.emit(itemRawString)
+		lx.next()
+		lx.ignore()
+		return lx.pop()
+	}
+	return lexRawString
+}
+
+// lexMultilineRawString consumes a raw string. Nothing can be escaped in such
+// a string. It assumes that the beginning "'''" has already been consumed and
+// ignored.
+func lexMultilineRawString(lx *lexer) stateFn {
+	switch lx.next() {
+	case eof:
+		return lx.errorf("unexpected EOF")
+	case rawStringEnd:
+		if lx.accept(rawStringEnd) {
+			if lx.accept(rawStringEnd) {
+				lx.backup()
+				lx.backup()
+				lx.backup()
+				lx.emit(itemRawMultilineString)
+				lx.next()
+				lx.next()
+				lx.next()
+				lx.ignore()
+				return lx.pop()
+			}
+			lx.backup()
+		}
+	}
+	return lexMultilineRawString
+}
+
+// lexMultilineStringEscape consumes an escaped character. It assumes that the
+// preceding '\\' has already been consumed.
+func lexMultilineStringEscape(lx *lexer) stateFn {
+	// Handle the special case first:
+	if isNL(lx.next()) {
+		return lexMultilineString
+	}
+	lx.backup()
+	lx.push(lexMultilineString)
+	return lexStringEscape(lx)
+}
+
+func lexStringEscape(lx *lexer) stateFn {
+	r := lx.next()
+	switch r {
+	case 'b':
+		fallthrough
+	case 't':
+		fallthrough
+	case 'n':
+		fallthrough
+	case 'f':
+		fallthrough
+	case 'r':
+		fallthrough
+	case '"':
+		fallthrough
+	case '\\':
+		return lx.pop()
+	case 'u':
+		return lexShortUnicodeEscape
+	case 'U':
+		return lexLongUnicodeEscape
+	}
+	return lx.errorf("invalid escape character %q; only the following "+
+		"escape characters are allowed: "+
+		`\b, \t, \n, \f, \r, \", \\, \uXXXX, and \UXXXXXXXX`, r)
+}
+
+func lexShortUnicodeEscape(lx *lexer) stateFn {
+	var r rune
+	for i := 0; i < 4; i++ {
+		r = lx.next()
+		if !isHexadecimal(r) {
+			return lx.errorf(`expected four hexadecimal digits after '\u', `+
+				"but got %q instead", lx.current())
+		}
+	}
+	return lx.pop()
+}
+
+func lexLongUnicodeEscape(lx *lexer) stateFn {
+	var r rune
+	for i := 0; i < 8; i++ {
+		r = lx.next()
+		if !isHexadecimal(r) {
+			return lx.errorf(`expected eight hexadecimal digits after '\U', `+
+				"but got %q instead", lx.current())
+		}
+	}
+	return lx.pop()
+}
+
+// lexNumberOrDateStart consumes either an integer, a float, or datetime.
+func lexNumberOrDateStart(lx *lexer) stateFn {
+	r := lx.next()
+	if isDigit(r) {
+		return lexNumberOrDate
+	}
+	switch r {
+	case '_':
+		return lexNumber
+	case 'e', 'E':
+		return lexFloat
+	case '.':
+		return lx.errorf("floats must start with a digit, not '.'")
+	}
+	return lx.errorf("expected a digit but got %q", r)
+}
+
+// lexNumberOrDate consumes either an integer, float or datetime.
+func lexNumberOrDate(lx *lexer) stateFn {
+	r := lx.next()
+	if isDigit(r) {
+		return lexNumberOrDate
+	}
+	switch r {
+	case '-':
+		return lexDatetime
+	case '_':
+		return lexNumber
+	case '.', 'e', 'E':
+		return lexFloat
+	}
+
+	lx.backup()
+	lx.emit(itemInteger)
+	return lx.pop()
+}
+
+// lexDatetime consumes a Datetime, to a first approximation.
+// The parser validates that it matches one of the accepted formats.
+func lexDatetime(lx *lexer) stateFn {
+	r := lx.next()
+	if isDigit(r) {
+		return lexDatetime
+	}
+	switch r {
+	case '-', 'T', ':', '.', 'Z', '+':
+		return lexDatetime
+	}
+
+	lx.backup()
+	lx.emit(itemDatetime)
+	return lx.pop()
+}
+
+// lexNumberStart consumes either an integer or a float. It assumes that a sign
+// has already been read, but that *no* digits have been consumed.
+// lexNumberStart will move to the appropriate integer or float states.
+func lexNumberStart(lx *lexer) stateFn {
+	// We MUST see a digit. Even floats have to start with a digit.
+	r := lx.next()
+	if !isDigit(r) {
+		if r == '.' {
+			return lx.errorf("floats must start with a digit, not '.'")
+		}
+		return lx.errorf("expected a digit but got %q", r)
+	}
+	return lexNumber
+}
+
+// lexNumber consumes an integer or a float after seeing the first digit.
+func lexNumber(lx *lexer) stateFn {
+	r := lx.next()
+	if isDigit(r) {
+		return lexNumber
+	}
+	switch r {
+	case '_':
+		return lexNumber
+	case '.', 'e', 'E':
+		return lexFloat
+	}
+
+	lx.backup()
+	lx.emit(itemInteger)
+	return lx.pop()
+}
+
+// lexFloat consumes the elements of a float. It allows any sequence of
+// float-like characters, so floats emitted by the lexer are only a first
+// approximation and must be validated by the parser.
+func lexFloat(lx *lexer) stateFn {
+	r := lx.next()
+	if isDigit(r) {
+		return lexFloat
+	}
+	switch r {
+	case '_', '.', '-', '+', 'e', 'E':
+		return lexFloat
+	}
+
+	lx.backup()
+	lx.emit(itemFloat)
+	return lx.pop()
+}
+
+// lexBool consumes a bool string: 'true' or 'false.
+func lexBool(lx *lexer) stateFn {
+	var rs []rune
+	for {
+		r := lx.next()
+		if !unicode.IsLetter(r) {
+			lx.backup()
+			break
+		}
+		rs = append(rs, r)
+	}
+	s := string(rs)
+	switch s {
+	case "true", "false":
+		lx.emit(itemBool)
+		return lx.pop()
+	}
+	return lx.errorf("expected value but found %q instead", s)
+}
+
+// lexCommentStart begins the lexing of a comment. It will emit
+// itemCommentStart and consume no characters, passing control to lexComment.
+func lexCommentStart(lx *lexer) stateFn {
+	lx.ignore()
+	lx.emit(itemCommentStart)
+	return lexComment
+}
+
+// lexComment lexes an entire comment. It assumes that '#' has been consumed.
+// It will consume *up to* the first newline character, and pass control
+// back to the last state on the stack.
+func lexComment(lx *lexer) stateFn {
+	r := lx.peek()
+	if isNL(r) || r == eof {
+		lx.emit(itemText)
+		return lx.pop()
+	}
+	lx.next()
+	return lexComment
+}
+
+// lexSkip ignores all slurped input and moves on to the next state.
+func lexSkip(lx *lexer, nextState stateFn) stateFn {
+	return func(lx *lexer) stateFn {
+		lx.ignore()
+		return nextState
+	}
+}
+
+// isWhitespace returns true if `r` is a whitespace character according
+// to the spec.
+func isWhitespace(r rune) bool {
+	return r == '\t' || r == ' '
+}
+
+func isNL(r rune) bool {
+	return r == '\n' || r == '\r'
+}
+
+func isDigit(r rune) bool {
+	return r >= '0' && r <= '9'
+}
+
+func isHexadecimal(r rune) bool {
+	return (r >= '0' && r <= '9') ||
+		(r >= 'a' && r <= 'f') ||
+		(r >= 'A' && r <= 'F')
+}
+
+func isBareKeyChar(r rune) bool {
+	return (r >= 'A' && r <= 'Z') ||
+		(r >= 'a' && r <= 'z') ||
+		(r >= '0' && r <= '9') ||
+		r == '_' ||
+		r == '-'
+}
+
+func (itype itemType) String() string {
+	switch itype {
+	case itemError:
+		return "Error"
+	case itemNIL:
+		return "NIL"
+	case itemEOF:
+		return "EOF"
+	case itemText:
+		return "Text"
+	case itemString, itemRawString, itemMultilineString, itemRawMultilineString:
+		return "String"
+	case itemBool:
+		return "Bool"
+	case itemInteger:
+		return "Integer"
+	case itemFloat:
+		return "Float"
+	case itemDatetime:
+		return "DateTime"
+	case itemTableStart:
+		return "TableStart"
+	case itemTableEnd:
+		return "TableEnd"
+	case itemKeyStart:
+		return "KeyStart"
+	case itemArray:
+		return "Array"
+	case itemArrayEnd:
+		return "ArrayEnd"
+	case itemCommentStart:
+		return "CommentStart"
+	}
+	panic(fmt.Sprintf("BUG: Unknown type '%d'.", int(itype)))
+}
+
+func (item item) String() string {
+	return fmt.Sprintf("(%s, %s)", item.typ.String(), item.val)
+}

vendor/github.com/BurntSushi/toml/parse.go

@@ -0,0 +1,592 @@
+package toml
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+	"unicode"
+	"unicode/utf8"
+)
+
+type parser struct {
+	mapping map[string]interface{}
+	types   map[string]tomlType
+	lx      *lexer
+
+	// A list of keys in the order that they appear in the TOML data.
+	ordered []Key
+
+	// the full key for the current hash in scope
+	context Key
+
+	// the base key name for everything except hashes
+	currentKey string
+
+	// rough approximation of line number
+	approxLine int
+
+	// A map of 'key.group.names' to whether they were created implicitly.
+	implicits map[string]bool
+}
+
+type parseError string
+
+func (pe parseError) Error() string {
+	return string(pe)
+}
+
+func parse(data string) (p *parser, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			var ok bool
+			if err, ok = r.(parseError); ok {
+				return
+			}
+			panic(r)
+		}
+	}()
+
+	p = &parser{
+		mapping:   make(map[string]interface{}),
+		types:     make(map[string]tomlType),
+		lx:        lex(data),
+		ordered:   make([]Key, 0),
+		implicits: make(map[string]bool),
+	}
+	for {
+		item := p.next()
+		if item.typ == itemEOF {
+			break
+		}
+		p.topLevel(item)
+	}
+
+	return p, nil
+}
+
+func (p *parser) panicf(format string, v ...interface{}) {
+	msg := fmt.Sprintf("Near line %d (last key parsed '%s'): %s",
+		p.approxLine, p.current(), fmt.Sprintf(format, v...))
+	panic(parseError(msg))
+}
+
+func (p *parser) next() item {
+	it := p.lx.nextItem()
+	if it.typ == itemError {
+		p.panicf("%s", it.val)
+	}
+	return it
+}
+
+func (p *parser) bug(format string, v ...interface{}) {
+	panic(fmt.Sprintf("BUG: "+format+"\n\n", v...))
+}
+
+func (p *parser) expect(typ itemType) item {
+	it := p.next()
+	p.assertEqual(typ, it.typ)
+	return it
+}
+
+func (p *parser) assertEqual(expected, got itemType) {
+	if expected != got {
+		p.bug("Expected '%s' but got '%s'.", expected, got)
+	}
+}
+
+func (p *parser) topLevel(item item) {
+	switch item.typ {
+	case itemCommentStart:
+		p.approxLine = item.line
+		p.expect(itemText)
+	case itemTableStart:
+		kg := p.next()
+		p.approxLine = kg.line
+
+		var key Key
+		for ; kg.typ != itemTableEnd && kg.typ != itemEOF; kg = p.next() {
+			key = append(key, p.keyString(kg))
+		}
+		p.assertEqual(itemTableEnd, kg.typ)
+
+		p.establishContext(key, false)
+		p.setType("", tomlHash)
+		p.ordered = append(p.ordered, key)
+	case itemArrayTableStart:
+		kg := p.next()
+		p.approxLine = kg.line
+
+		var key Key
+		for ; kg.typ != itemArrayTableEnd && kg.typ != itemEOF; kg = p.next() {
+			key = append(key, p.keyString(kg))
+		}
+		p.assertEqual(itemArrayTableEnd, kg.typ)
+
+		p.establishContext(key, true)
+		p.setType("", tomlArrayHash)
+		p.ordered = append(p.ordered, key)
+	case itemKeyStart:
+		kname := p.next()
+		p.approxLine = kname.line
+		p.currentKey = p.keyString(kname)
+
+		val, typ := p.value(p.next())
+		p.setValue(p.currentKey, val)
+		p.setType(p.currentKey, typ)
+		p.ordered = append(p.ordered, p.context.add(p.currentKey))
+		p.currentKey = ""
+	default:
+		p.bug("Unexpected type at top level: %s", item.typ)
+	}
+}
+
+// Gets a string for a key (or part of a key in a table name).
+func (p *parser) keyString(it item) string {
+	switch it.typ {
+	case itemText:
+		return it.val
+	case itemString, itemMultilineString,
+		itemRawString, itemRawMultilineString:
+		s, _ := p.value(it)
+		return s.(string)
+	default:
+		p.bug("Unexpected key type: %s", it.typ)
+		panic("unreachable")
+	}
+}
+
+// value translates an expected value from the lexer into a Go value wrapped
+// as an empty interface.
+func (p *parser) value(it item) (interface{}, tomlType) {
+	switch it.typ {
+	case itemString:
+		return p.replaceEscapes(it.val), p.typeOfPrimitive(it)
+	case itemMultilineString:
+		trimmed := stripFirstNewline(stripEscapedWhitespace(it.val))
+		return p.replaceEscapes(trimmed), p.typeOfPrimitive(it)
+	case itemRawString:
+		return it.val, p.typeOfPrimitive(it)
+	case itemRawMultilineString:
+		return stripFirstNewline(it.val), p.typeOfPrimitive(it)
+	case itemBool:
+		switch it.val {
+		case "true":
+			return true, p.typeOfPrimitive(it)
+		case "false":
+			return false, p.typeOfPrimitive(it)
+		}
+		p.bug("Expected boolean value, but got '%s'.", it.val)
+	case itemInteger:
+		if !numUnderscoresOK(it.val) {
+			p.panicf("Invalid integer %q: underscores must be surrounded by digits",
+				it.val)
+		}
+		val := strings.Replace(it.val, "_", "", -1)
+		num, err := strconv.ParseInt(val, 10, 64)
+		if err != nil {
+			// Distinguish integer values. Normally, it'd be a bug if the lexer
+			// provides an invalid integer, but it's possible that the number is
+			// out of range of valid values (which the lexer cannot determine).
+			// So mark the former as a bug but the latter as a legitimate user
+			// error.
+			if e, ok := err.(*strconv.NumError); ok &&
+				e.Err == strconv.ErrRange {
+
+				p.panicf("Integer '%s' is out of the range of 64-bit "+
+					"signed integers.", it.val)
+			} else {
+				p.bug("Expected integer value, but got '%s'.", it.val)
+			}
+		}
+		return num, p.typeOfPrimitive(it)
+	case itemFloat:
+		parts := strings.FieldsFunc(it.val, func(r rune) bool {
+			switch r {
+			case '.', 'e', 'E':
+				return true
+			}
+			return false
+		})
+		for _, part := range parts {
+			if !numUnderscoresOK(part) {
+				p.panicf("Invalid float %q: underscores must be "+
+					"surrounded by digits", it.val)
+			}
+		}
+		if !numPeriodsOK(it.val) {
+			// As a special case, numbers like '123.' or '1.e2',
+			// which are valid as far as Go/strconv are concerned,
+			// must be rejected because TOML says that a fractional
+			// part consists of '.' followed by 1+ digits.
+			p.panicf("Invalid float %q: '.' must be followed "+
+				"by one or more digits", it.val)
+		}
+		val := strings.Replace(it.val, "_", "", -1)
+		num, err := strconv.ParseFloat(val, 64)
+		if err != nil {
+			if e, ok := err.(*strconv.NumError); ok &&
+				e.Err == strconv.ErrRange {
+
+				p.panicf("Float '%s' is out of the range of 64-bit "+
+					"IEEE-754 floating-point numbers.", it.val)
+			} else {
+				p.panicf("Invalid float value: %q", it.val)
+			}
+		}
+		return num, p.typeOfPrimitive(it)
+	case itemDatetime:
+		var t time.Time
+		var ok bool
+		var err error
+		for _, format := range []string{
+			"2006-01-02T15:04:05Z07:00",
+			"2006-01-02T15:04:05",
+			"2006-01-02",
+		} {
+			t, err = time.ParseInLocation(format, it.val, time.Local)
+			if err == nil {
+				ok = true
+				break
+			}
+		}
+		if !ok {
+			p.panicf("Invalid TOML Datetime: %q.", it.val)
+		}
+		return t, p.typeOfPrimitive(it)
+	case itemArray:
+		array := make([]interface{}, 0)
+		types := make([]tomlType, 0)
+
+		for it = p.next(); it.typ != itemArrayEnd; it = p.next() {
+			if it.typ == itemCommentStart {
+				p.expect(itemText)
+				continue
+			}
+
+			val, typ := p.value(it)
+			array = append(array, val)
+			types = append(types, typ)
+		}
+		return array, p.typeOfArray(types)
+	case itemInlineTableStart:
+		var (
+			hash         = make(map[string]interface{})
+			outerContext = p.context
+			outerKey     = p.currentKey
+		)
+
+		p.context = append(p.context, p.currentKey)
+		p.currentKey = ""
+		for it := p.next(); it.typ != itemInlineTableEnd; it = p.next() {
+			if it.typ != itemKeyStart {
+				p.bug("Expected key start but instead found %q, around line %d",
+					it.val, p.approxLine)
+			}
+			if it.typ == itemCommentStart {
+				p.expect(itemText)
+				continue
+			}
+
+			// retrieve key
+			k := p.next()
+			p.approxLine = k.line
+			kname := p.keyString(k)
+
+			// retrieve value
+			p.currentKey = kname
+			val, typ := p.value(p.next())
+			// make sure we keep metadata up to date
+			p.setType(kname, typ)
+			p.ordered = append(p.ordered, p.context.add(p.currentKey))
+			hash[kname] = val
+		}
+		p.context = outerContext
+		p.currentKey = outerKey
+		return hash, tomlHash
+	}
+	p.bug("Unexpected value type: %s", it.typ)
+	panic("unreachable")
+}
+
+// numUnderscoresOK checks whether each underscore in s is surrounded by
+// characters that are not underscores.
+func numUnderscoresOK(s string) bool {
+	accept := false
+	for _, r := range s {
+		if r == '_' {
+			if !accept {
+				return false
+			}
+			accept = false
+			continue
+		}
+		accept = true
+	}
+	return accept
+}
+
+// numPeriodsOK checks whether every period in s is followed by a digit.
+func numPeriodsOK(s string) bool {
+	period := false
+	for _, r := range s {
+		if period && !isDigit(r) {
+			return false
+		}
+		period = r == '.'
+	}
+	return !period
+}
+
+// establishContext sets the current context of the parser,
+// where the context is either a hash or an array of hashes. Which one is
+// set depends on the value of the `array` parameter.
+//
+// Establishing the context also makes sure that the key isn't a duplicate, and
+// will create implicit hashes automatically.
+func (p *parser) establishContext(key Key, array bool) {
+	var ok bool
+
+	// Always start at the top level and drill down for our context.
+	hashContext := p.mapping
+	keyContext := make(Key, 0)
+
+	// We only need implicit hashes for key[0:-1]
+	for _, k := range key[0 : len(key)-1] {
+		_, ok = hashContext[k]
+		keyContext = append(keyContext, k)
+
+		// No key? Make an implicit hash and move on.
+		if !ok {
+			p.addImplicit(keyContext)
+			hashContext[k] = make(map[string]interface{})
+		}
+
+		// If the hash context is actually an array of tables, then set
+		// the hash context to the last element in that array.
+		//
+		// Otherwise, it better be a table, since this MUST be a key group (by
+		// virtue of it not being the last element in a key).
+		switch t := hashContext[k].(type) {
+		case []map[string]interface{}:
+			hashContext = t[len(t)-1]
+		case map[string]interface{}:
+			hashContext = t
+		default:
+			p.panicf("Key '%s' was already created as a hash.", keyContext)
+		}
+	}
+
+	p.context = keyContext
+	if array {
+		// If this is the first element for this array, then allocate a new
+		// list of tables for it.
+		k := key[len(key)-1]
+		if _, ok := hashContext[k]; !ok {
+			hashContext[k] = make([]map[string]interface{}, 0, 5)
+		}
+
+		// Add a new table. But make sure the key hasn't already been used
+		// for something else.
+		if hash, ok := hashContext[k].([]map[string]interface{}); ok {
+			hashContext[k] = append(hash, make(map[string]interface{}))
+		} else {
+			p.panicf("Key '%s' was already created and cannot be used as "+
+				"an array.", keyContext)
+		}
+	} else {
+		p.setValue(key[len(key)-1], make(map[string]interface{}))
+	}
+	p.context = append(p.context, key[len(key)-1])
+}
+
+// setValue sets the given key to the given value in the current context.
+// It will make sure that the key hasn't already been defined, account for
+// implicit key groups.
+func (p *parser) setValue(key string, value interface{}) {
+	var tmpHash interface{}
+	var ok bool
+
+	hash := p.mapping
+	keyContext := make(Key, 0)
+	for _, k := range p.context {
+		keyContext = append(keyContext, k)
+		if tmpHash, ok = hash[k]; !ok {
+			p.bug("Context for key '%s' has not been established.", keyContext)
+		}
+		switch t := tmpHash.(type) {
+		case []map[string]interface{}:
+			// The context is a table of hashes. Pick the most recent table
+			// defined as the current hash.
+			hash = t[len(t)-1]
+		case map[string]interface{}:
+			hash = t
+		default:
+			p.bug("Expected hash to have type 'map[string]interface{}', but "+
+				"it has '%T' instead.", tmpHash)
+		}
+	}
+	keyContext = append(keyContext, key)
+
+	if _, ok := hash[key]; ok {
+		// Typically, if the given key has already been set, then we have
+		// to raise an error since duplicate keys are disallowed. However,
+		// it's possible that a key was previously defined implicitly. In this
+		// case, it is allowed to be redefined concretely. (See the
+		// `tests/valid/implicit-and-explicit-after.toml` test in `toml-test`.)
+		//
+		// But we have to make sure to stop marking it as an implicit. (So that
+		// another redefinition provokes an error.)
+		//
+		// Note that since it has already been defined (as a hash), we don't
+		// want to overwrite it. So our business is done.
+		if p.isImplicit(keyContext) {
+			p.removeImplicit(keyContext)
+			return
+		}
+
+		// Otherwise, we have a concrete key trying to override a previous
+		// key, which is *always* wrong.
+		p.panicf("Key '%s' has already been defined.", keyContext)
+	}
+	hash[key] = value
+}
+
+// setType sets the type of a particular value at a given key.
+// It should be called immediately AFTER setValue.
+//
+// Note that if `key` is empty, then the type given will be applied to the
+// current context (which is either a table or an array of tables).
+func (p *parser) setType(key string, typ tomlType) {
+	keyContext := make(Key, 0, len(p.context)+1)
+	for _, k := range p.context {
+		keyContext = append(keyContext, k)
+	}
+	if len(key) > 0 { // allow type setting for hashes
+		keyContext = append(keyContext, key)
+	}
+	p.types[keyContext.String()] = typ
+}
+
+// addImplicit sets the given Key as having been created implicitly.
+func (p *parser) addImplicit(key Key) {
+	p.implicits[key.String()] = true
+}
+
+// removeImplicit stops tagging the given key as having been implicitly
+// created.
+func (p *parser) removeImplicit(key Key) {
+	p.implicits[key.String()] = false
+}
+
+// isImplicit returns true if the key group pointed to by the key was created
+// implicitly.
+func (p *parser) isImplicit(key Key) bool {
+	return p.implicits[key.String()]
+}
+
+// current returns the full key name of the current context.
+func (p *parser) current() string {
+	if len(p.currentKey) == 0 {
+		return p.context.String()
+	}
+	if len(p.context) == 0 {
+		return p.currentKey
+	}
+	return fmt.Sprintf("%s.%s", p.context, p.currentKey)
+}
+
+func stripFirstNewline(s string) string {
+	if len(s) == 0 || s[0] != '\n' {
+		return s
+	}
+	return s[1:]
+}
+
+func stripEscapedWhitespace(s string) string {
+	esc := strings.Split(s, "\\\n")
+	if len(esc) > 1 {
+		for i := 1; i < len(esc); i++ {
+			esc[i] = strings.TrimLeftFunc(esc[i], unicode.IsSpace)
+		}
+	}
+	return strings.Join(esc, "")
+}
+
+func (p *parser) replaceEscapes(str string) string {
+	var replaced []rune
+	s := []byte(str)
+	r := 0
+	for r < len(s) {
+		if s[r] != '\\' {
+			c, size := utf8.DecodeRune(s[r:])
+			r += size
+			replaced = append(replaced, c)
+			continue
+		}
+		r += 1
+		if r >= len(s) {
+			p.bug("Escape sequence at end of string.")
+			return ""
+		}
+		switch s[r] {
+		default:
+			p.bug("Expected valid escape code after \\, but got %q.", s[r])
+			return ""
+		case 'b':
+			replaced = append(replaced, rune(0x0008))
+			r += 1
+		case 't':
+			replaced = append(replaced, rune(0x0009))
+			r += 1
+		case 'n':
+			replaced = append(replaced, rune(0x000A))
+			r += 1
+		case 'f':
+			replaced = append(replaced, rune(0x000C))
+			r += 1
+		case 'r':
+			replaced = append(replaced, rune(0x000D))
+			r += 1
+		case '"':
+			replaced = append(replaced, rune(0x0022))
+			r += 1
+		case '\\':
+			replaced = append(replaced, rune(0x005C))
+			r += 1
+		case 'u':
+			// At this point, we know we have a Unicode escape of the form
+			// `uXXXX` at [r, r+5). (Because the lexer guarantees this
+			// for us.)
+			escaped := p.asciiEscapeToUnicode(s[r+1 : r+5])
+			replaced = append(replaced, escaped)
+			r += 5
+		case 'U':
+			// At this point, we know we have a Unicode escape of the form
+			// `uXXXX` at [r, r+9). (Because the lexer guarantees this
+			// for us.)
+			escaped := p.asciiEscapeToUnicode(s[r+1 : r+9])
+			replaced = append(replaced, escaped)
+			r += 9
+		}
+	}
+	return string(replaced)
+}
+
+func (p *parser) asciiEscapeToUnicode(bs []byte) rune {
+	s := string(bs)
+	hex, err := strconv.ParseUint(strings.ToLower(s), 16, 32)
+	if err != nil {
+		p.bug("Could not parse '%s' as a hexadecimal number, but the "+
+			"lexer claims it's OK: %s", s, err)
+	}
+	if !utf8.ValidRune(rune(hex)) {
+		p.panicf("Escaped character '\\u%s' is not valid UTF-8.", s)
+	}
+	return rune(hex)
+}
+
+func isStringType(ty itemType) bool {
+	return ty == itemString || ty == itemMultilineString ||
+		ty == itemRawString || ty == itemRawMultilineString
+}

vendor/github.com/BurntSushi/toml/type_check.go

@@ -0,0 +1,91 @@
+package toml
+
+// tomlType represents any Go type that corresponds to a TOML type.
+// While the first draft of the TOML spec has a simplistic type system that
+// probably doesn't need this level of sophistication, we seem to be militating
+// toward adding real composite types.
+type tomlType interface {
+	typeString() string
+}
+
+// typeEqual accepts any two types and returns true if they are equal.
+func typeEqual(t1, t2 tomlType) bool {
+	if t1 == nil || t2 == nil {
+		return false
+	}
+	return t1.typeString() == t2.typeString()
+}
+
+func typeIsHash(t tomlType) bool {
+	return typeEqual(t, tomlHash) || typeEqual(t, tomlArrayHash)
+}
+
+type tomlBaseType string
+
+func (btype tomlBaseType) typeString() string {
+	return string(btype)
+}
+
+func (btype tomlBaseType) String() string {
+	return btype.typeString()
+}
+
+var (
+	tomlInteger   tomlBaseType = "Integer"
+	tomlFloat     tomlBaseType = "Float"
+	tomlDatetime  tomlBaseType = "Datetime"
+	tomlString    tomlBaseType = "String"
+	tomlBool      tomlBaseType = "Bool"
+	tomlArray     tomlBaseType = "Array"
+	tomlHash      tomlBaseType = "Hash"
+	tomlArrayHash tomlBaseType = "ArrayHash"
+)
+
+// typeOfPrimitive returns a tomlType of any primitive value in TOML.
+// Primitive values are: Integer, Float, Datetime, String and Bool.
+//
+// Passing a lexer item other than the following will cause a BUG message
+// to occur: itemString, itemBool, itemInteger, itemFloat, itemDatetime.
+func (p *parser) typeOfPrimitive(lexItem item) tomlType {
+	switch lexItem.typ {
+	case itemInteger:
+		return tomlInteger
+	case itemFloat:
+		return tomlFloat
+	case itemDatetime:
+		return tomlDatetime
+	case itemString:
+		return tomlString
+	case itemMultilineString:
+		return tomlString
+	case itemRawString:
+		return tomlString
+	case itemRawMultilineString:
+		return tomlString
+	case itemBool:
+		return tomlBool
+	}
+	p.bug("Cannot infer primitive type of lex item '%s'.", lexItem)
+	panic("unreachable")
+}
+
+// typeOfArray returns a tomlType for an array given a list of types of its
+// values.
+//
+// In the current spec, if an array is homogeneous, then its type is always
+// "Array". If the array is not homogeneous, an error is generated.
+func (p *parser) typeOfArray(types []tomlType) tomlType {
+	// Empty arrays are cool.
+	if len(types) == 0 {
+		return tomlArray
+	}
+
+	theType := types[0]
+	for _, t := range types[1:] {
+		if !typeEqual(theType, t) {
+			p.panicf("Array contains values of type '%s' and '%s', but "+
+				"arrays must be homogeneous.", theType, t)
+		}
+	}
+	return tomlArray
+}

vendor/github.com/BurntSushi/toml/type_fields.go

@@ -0,0 +1,242 @@
+package toml
+
+// Struct field handling is adapted from code in encoding/json:
+//
+// Copyright 2010 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the Go distribution.
+
+import (
+	"reflect"
+	"sort"
+	"sync"
+)
+
+// A field represents a single field found in a struct.
+type field struct {
+	name  string       // the name of the field (`toml` tag included)
+	tag   bool         // whether field has a `toml` tag
+	index []int        // represents the depth of an anonymous field
+	typ   reflect.Type // the type of the field
+}
+
+// byName sorts field by name, breaking ties with depth,
+// then breaking ties with "name came from toml tag", then
+// breaking ties with index sequence.
+type byName []field
+
+func (x byName) Len() int { return len(x) }
+
+func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
+
+func (x byName) Less(i, j int) bool {
+	if x[i].name != x[j].name {
+		return x[i].name < x[j].name
+	}
+	if len(x[i].index) != len(x[j].index) {
+		return len(x[i].index) < len(x[j].index)
+	}
+	if x[i].tag != x[j].tag {
+		return x[i].tag
+	}
+	return byIndex(x).Less(i, j)
+}
+
+// byIndex sorts field by index sequence.
+type byIndex []field
+
+func (x byIndex) Len() int { return len(x) }
+
+func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
+
+func (x byIndex) Less(i, j int) bool {
+	for k, xik := range x[i].index {
+		if k >= len(x[j].index) {
+			return false
+		}
+		if xik != x[j].index[k] {
+			return xik < x[j].index[k]
+		}
+	}
+	return len(x[i].index) < len(x[j].index)
+}
+
+// typeFields returns a list of fields that TOML should recognize for the given
+// type. The algorithm is breadth-first search over the set of structs to
+// include - the top struct and then any reachable anonymous structs.
+func typeFields(t reflect.Type) []field {
+	// Anonymous fields to explore at the current level and the next.
+	current := []field{}
+	next := []field{{typ: t}}
+
+	// Count of queued names for current level and the next.
+	count := map[reflect.Type]int{}
+	nextCount := map[reflect.Type]int{}
+
+	// Types already visited at an earlier level.
+	visited := map[reflect.Type]bool{}
+
+	// Fields found.
+	var fields []field
+
+	for len(next) > 0 {
+		current, next = next, current[:0]
+		count, nextCount = nextCount, map[reflect.Type]int{}
+
+		for _, f := range current {
+			if visited[f.typ] {
+				continue
+			}
+			visited[f.typ] = true
+
+			// Scan f.typ for fields to include.
+			for i := 0; i < f.typ.NumField(); i++ {
+				sf := f.typ.Field(i)
+				if sf.PkgPath != "" && !sf.Anonymous { // unexported
+					continue
+				}
+				opts := getOptions(sf.Tag)
+				if opts.skip {
+					continue
+				}
+				index := make([]int, len(f.index)+1)
+				copy(index, f.index)
+				index[len(f.index)] = i
+
+				ft := sf.Type
+				if ft.Name() == "" && ft.Kind() == reflect.Ptr {
+					// Follow pointer.
+					ft = ft.Elem()
+				}
+
+				// Record found field and index sequence.
+				if opts.name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct {
+					tagged := opts.name != ""
+					name := opts.name
+					if name == "" {
+						name = sf.Name
+					}
+					fields = append(fields, field{name, tagged, index, ft})
+					if count[f.typ] > 1 {
+						// If there were multiple instances, add a second,
+						// so that the annihilation code will see a duplicate.
+						// It only cares about the distinction between 1 or 2,
+						// so don't bother generating any more copies.
+						fields = append(fields, fields[len(fields)-1])
+					}
+					continue
+				}
+
+				// Record new anonymous struct to explore in next round.
+				nextCount[ft]++
+				if nextCount[ft] == 1 {
+					f := field{name: ft.Name(), index: index, typ: ft}
+					next = append(next, f)
+				}
+			}
+		}
+	}
+
+	sort.Sort(byName(fields))
+
+	// Delete all fields that are hidden by the Go rules for embedded fields,
+	// except that fields with TOML tags are promoted.
+
+	// The fields are sorted in primary order of name, secondary order
+	// of field index length. Loop over names; for each name, delete
+	// hidden fields by choosing the one dominant field that survives.
+	out := fields[:0]
+	for advance, i := 0, 0; i < len(fields); i += advance {
+		// One iteration per name.
+		// Find the sequence of fields with the name of this first field.
+		fi := fields[i]
+		name := fi.name
+		for advance = 1; i+advance < len(fields); advance++ {
+			fj := fields[i+advance]
+			if fj.name != name {
+				break
+			}
+		}
+		if advance == 1 { // Only one field with this name
+			out = append(out, fi)
+			continue
+		}
+		dominant, ok := dominantField(fields[i : i+advance])
+		if ok {
+			out = append(out, dominant)
+		}
+	}
+
+	fields = out
+	sort.Sort(byIndex(fields))
+
+	return fields
+}
+
+// dominantField looks through the fields, all of which are known to
+// have the same name, to find the single field that dominates the
+// others using Go's embedding rules, modified by the presence of
+// TOML tags. If there are multiple top-level fields, the boolean
+// will be false: This condition is an error in Go and we skip all
+// the fields.
+func dominantField(fields []field) (field, bool) {
+	// The fields are sorted in increasing index-length order. The winner
+	// must therefore be one with the shortest index length. Drop all
+	// longer entries, which is easy: just truncate the slice.
+	length := len(fields[0].index)
+	tagged := -1 // Index of first tagged field.
+	for i, f := range fields {
+		if len(f.index) > length {
+			fields = fields[:i]
+			break
+		}
+		if f.tag {
+			if tagged >= 0 {
+				// Multiple tagged fields at the same level: conflict.
+				// Return no field.
+				return field{}, false
+			}
+			tagged = i
+		}
+	}
+	if tagged >= 0 {
+		return fields[tagged], true
+	}
+	// All remaining fields have the same length. If there's more than one,
+	// we have a conflict (two fields named "X" at the same level) and we
+	// return no field.
+	if len(fields) > 1 {
+		return field{}, false
+	}
+	return fields[0], true
+}
+
+var fieldCache struct {
+	sync.RWMutex
+	m map[reflect.Type][]field
+}
+
+// cachedTypeFields is like typeFields but uses a cache to avoid repeated work.
+func cachedTypeFields(t reflect.Type) []field {
+	fieldCache.RLock()
+	f := fieldCache.m[t]
+	fieldCache.RUnlock()
+	if f != nil {
+		return f
+	}
+
+	// Compute fields without lock.
+	// Might duplicate effort but won't hold other computations back.
+	f = typeFields(t)
+	if f == nil {
+		f = []field{}
+	}
+
+	fieldCache.Lock()
+	if fieldCache.m == nil {
+		fieldCache.m = map[reflect.Type][]field{}
+	}
+	fieldCache.m[t] = f
+	fieldCache.Unlock()
+	return f
+}

vendor/github.com/couchbase/gomemcached/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2013 Dustin Sallings
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/couchbase/gomemcached/client/tap_feed.go

@@ -0,0 +1,333 @@
+package memcached
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"math"
+
+	"github.com/couchbase/gomemcached"
+	"github.com/couchbase/goutils/logging"
+)
+
+// TAP protocol docs: <http://www.couchbase.com/wiki/display/couchbase/TAP+Protocol>
+
+// TapOpcode is the tap operation type (found in TapEvent)
+type TapOpcode uint8
+
+// Tap opcode values.
+const (
+	TapBeginBackfill = TapOpcode(iota)
+	TapEndBackfill
+	TapMutation
+	TapDeletion
+	TapCheckpointStart
+	TapCheckpointEnd
+	tapEndStream
+)
+
+const tapMutationExtraLen = 16
+
+var tapOpcodeNames map[TapOpcode]string
+
+func init() {
+	tapOpcodeNames = map[TapOpcode]string{
+		TapBeginBackfill:   "BeginBackfill",
+		TapEndBackfill:     "EndBackfill",
+		TapMutation:        "Mutation",
+		TapDeletion:        "Deletion",
+		TapCheckpointStart: "TapCheckpointStart",
+		TapCheckpointEnd:   "TapCheckpointEnd",
+		tapEndStream:       "EndStream",
+	}
+}
+
+func (opcode TapOpcode) String() string {
+	name := tapOpcodeNames[opcode]
+	if name == "" {
+		name = fmt.Sprintf("#%d", opcode)
+	}
+	return name
+}
+
+// TapEvent is a TAP notification of an operation on the server.
+type TapEvent struct {
+	Opcode     TapOpcode // Type of event
+	VBucket    uint16    // VBucket this event applies to
+	Flags      uint32    // Item flags
+	Expiry     uint32    // Item expiration time
+	Key, Value []byte    // Item key/value
+	Cas        uint64
+}
+
+func makeTapEvent(req gomemcached.MCRequest) *TapEvent {
+	event := TapEvent{
+		VBucket: req.VBucket,
+	}
+	switch req.Opcode {
+	case gomemcached.TAP_MUTATION:
+		event.Opcode = TapMutation
+		event.Key = req.Key
+		event.Value = req.Body
+		event.Cas = req.Cas
+	case gomemcached.TAP_DELETE:
+		event.Opcode = TapDeletion
+		event.Key = req.Key
+		event.Cas = req.Cas
+	case gomemcached.TAP_CHECKPOINT_START:
+		event.Opcode = TapCheckpointStart
+	case gomemcached.TAP_CHECKPOINT_END:
+		event.Opcode = TapCheckpointEnd
+	case gomemcached.TAP_OPAQUE:
+		if len(req.Extras) < 8+4 {
+			return nil
+		}
+		switch op := int(binary.BigEndian.Uint32(req.Extras[8:])); op {
+		case gomemcached.TAP_OPAQUE_INITIAL_VBUCKET_STREAM:
+			event.Opcode = TapBeginBackfill
+		case gomemcached.TAP_OPAQUE_CLOSE_BACKFILL:
+			event.Opcode = TapEndBackfill
+		case gomemcached.TAP_OPAQUE_CLOSE_TAP_STREAM:
+			event.Opcode = tapEndStream
+		case gomemcached.TAP_OPAQUE_ENABLE_AUTO_NACK:
+			return nil
+		case gomemcached.TAP_OPAQUE_ENABLE_CHECKPOINT_SYNC:
+			return nil
+		default:
+			logging.Infof("TapFeed: Ignoring TAP_OPAQUE/%d", op)
+			return nil // unknown opaque event
+		}
+	case gomemcached.NOOP:
+		return nil // ignore
+	default:
+		logging.Infof("TapFeed: Ignoring %s", req.Opcode)
+		return nil // unknown event
+	}
+
+	if len(req.Extras) >= tapMutationExtraLen &&
+		(event.Opcode == TapMutation || event.Opcode == TapDeletion) {
+
+		event.Flags = binary.BigEndian.Uint32(req.Extras[8:])
+		event.Expiry = binary.BigEndian.Uint32(req.Extras[12:])
+	}
+
+	return &event
+}
+
+func (event TapEvent) String() string {
+	switch event.Opcode {
+	case TapBeginBackfill, TapEndBackfill, TapCheckpointStart, TapCheckpointEnd:
+		return fmt.Sprintf("<TapEvent %s, vbucket=%d>",
+			event.Opcode, event.VBucket)
+	default:
+		return fmt.Sprintf("<TapEvent %s, key=%q (%d bytes) flags=%x, exp=%d>",
+			event.Opcode, event.Key, len(event.Value),
+			event.Flags, event.Expiry)
+	}
+}
+
+// TapArguments are parameters for requesting a TAP feed.
+//
+// Call DefaultTapArguments to get a default one.
+type TapArguments struct {
+	// Timestamp of oldest item to send.
+	//
+	// Use TapNoBackfill to suppress all past items.
+	Backfill uint64
+	// If set, server will disconnect after sending existing items.
+	Dump bool
+	// The indices of the vbuckets to watch; empty/nil to watch all.
+	VBuckets []uint16
+	// Transfers ownership of vbuckets during cluster rebalance.
+	Takeover bool
+	// If true, server will wait for client ACK after every notification.
+	SupportAck bool
+	// If true, client doesn't want values so server shouldn't send them.
+	KeysOnly bool
+	// If true, client wants the server to send checkpoint events.
+	Checkpoint bool
+	// Optional identifier to use for this client, to allow reconnects
+	ClientName string
+	// Registers this client (by name) till explicitly deregistered.
+	RegisteredClient bool
+}
+
+// Value for TapArguments.Backfill denoting that no past events at all
+// should be sent.
+const TapNoBackfill = math.MaxUint64
+
+// DefaultTapArguments returns a default set of parameter values to
+// pass to StartTapFeed.
+func DefaultTapArguments() TapArguments {
+	return TapArguments{
+		Backfill: TapNoBackfill,
+	}
+}
+
+func (args *TapArguments) flags() []byte {
+	var flags gomemcached.TapConnectFlag
+	if args.Backfill != 0 {
+		flags |= gomemcached.BACKFILL
+	}
+	if args.Dump {
+		flags |= gomemcached.DUMP
+	}
+	if len(args.VBuckets) > 0 {
+		flags |= gomemcached.LIST_VBUCKETS
+	}
+	if args.Takeover {
+		flags |= gomemcached.TAKEOVER_VBUCKETS
+	}
+	if args.SupportAck {
+		flags |= gomemcached.SUPPORT_ACK
+	}
+	if args.KeysOnly {
+		flags |= gomemcached.REQUEST_KEYS_ONLY
+	}
+	if args.Checkpoint {
+		flags |= gomemcached.CHECKPOINT
+	}
+	if args.RegisteredClient {
+		flags |= gomemcached.REGISTERED_CLIENT
+	}
+	encoded := make([]byte, 4)
+	binary.BigEndian.PutUint32(encoded, uint32(flags))
+	return encoded
+}
+
+func must(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (args *TapArguments) bytes() (rv []byte) {
+	buf := bytes.NewBuffer([]byte{})
+
+	if args.Backfill > 0 {
+		must(binary.Write(buf, binary.BigEndian, uint64(args.Backfill)))
+	}
+
+	if len(args.VBuckets) > 0 {
+		must(binary.Write(buf, binary.BigEndian, uint16(len(args.VBuckets))))
+		for i := 0; i < len(args.VBuckets); i++ {
+			must(binary.Write(buf, binary.BigEndian, uint16(args.VBuckets[i])))
+		}
+	}
+	return buf.Bytes()
+}
+
+// TapFeed represents a stream of events from a server.
+type TapFeed struct {
+	C      <-chan TapEvent
+	Error  error
+	closer chan bool
+}
+
+// StartTapFeed starts a TAP feed on a client connection.
+//
+// The events can be read from the returned channel.  The connection
+// can no longer be used for other purposes; it's now reserved for
+// receiving the TAP messages. To stop receiving events, close the
+// client connection.
+func (mc *Client) StartTapFeed(args TapArguments) (*TapFeed, error) {
+	rq := &gomemcached.MCRequest{
+		Opcode: gomemcached.TAP_CONNECT,
+		Key:    []byte(args.ClientName),
+		Extras: args.flags(),
+		Body:   args.bytes()}
+
+	err := mc.Transmit(rq)
+	if err != nil {
+		return nil, err
+	}
+
+	ch := make(chan TapEvent)
+	feed := &TapFeed{
+		C:      ch,
+		closer: make(chan bool),
+	}
+	go mc.runFeed(ch, feed)
+	return feed, nil
+}
+
+// TapRecvHook is called after every incoming tap packet is received.
+var TapRecvHook func(*gomemcached.MCRequest, int, error)
+
+// Internal goroutine that reads from the socket and writes events to
+// the channel
+func (mc *Client) runFeed(ch chan TapEvent, feed *TapFeed) {
+	defer close(ch)
+	var headerBuf [gomemcached.HDR_LEN]byte
+loop:
+	for {
+		// Read the next request from the server.
+		//
+		//  (Can't call mc.Receive() because it reads a
+		//  _response_ not a request.)
+		var pkt gomemcached.MCRequest
+		n, err := pkt.Receive(mc.conn, headerBuf[:])
+		if TapRecvHook != nil {
+			TapRecvHook(&pkt, n, err)
+		}
+
+		if err != nil {
+			if err != io.EOF {
+				feed.Error = err
+			}
+			break loop
+		}
+
+		//logging.Infof("** TapFeed received %#v : %q", pkt, pkt.Body)
+
+		if pkt.Opcode == gomemcached.TAP_CONNECT {
+			// This is not an event from the server; it's
+			// an error response to my connect request.
+			feed.Error = fmt.Errorf("tap connection failed: %s", pkt.Body)
+			break loop
+		}
+
+		event := makeTapEvent(pkt)
+		if event != nil {
+			if event.Opcode == tapEndStream {
+				break loop
+			}
+
+			select {
+			case ch <- *event:
+			case <-feed.closer:
+				break loop
+			}
+		}
+
+		if len(pkt.Extras) >= 4 {
+			reqFlags := binary.BigEndian.Uint16(pkt.Extras[2:])
+			if reqFlags&gomemcached.TAP_ACK != 0 {
+				if _, err := mc.sendAck(&pkt); err != nil {
+					feed.Error = err
+					break loop
+				}
+			}
+		}
+	}
+	if err := mc.Close(); err != nil {
+		logging.Errorf("Error closing memcached client:  %v", err)
+	}
+}
+
+func (mc *Client) sendAck(pkt *gomemcached.MCRequest) (int, error) {
+	res := gomemcached.MCResponse{
+		Opcode: pkt.Opcode,
+		Opaque: pkt.Opaque,
+		Status: gomemcached.SUCCESS,
+	}
+	return res.Transmit(mc.conn)
+}
+
+// Close terminates a TapFeed.
+//
+//  Call this if you stop using a TapFeed before its channel ends.
+func (feed *TapFeed) Close() {
+	close(feed.closer)
+}

vendor/github.com/couchbase/gomemcached/client/transport.go

@@ -0,0 +1,67 @@
+package memcached
+
+import (
+	"errors"
+	"io"
+
+	"github.com/couchbase/gomemcached"
+)
+
+var errNoConn = errors.New("no connection")
+
+// UnwrapMemcachedError converts memcached errors to normal responses.
+//
+// If the error is a memcached response, declare the error to be nil
+// so a client can handle the status without worrying about whether it
+// indicates success or failure.
+func UnwrapMemcachedError(rv *gomemcached.MCResponse,
+	err error) (*gomemcached.MCResponse, error) {
+
+	if rv == err {
+		return rv, nil
+	}
+	return rv, err
+}
+
+// ReceiveHook is called after every packet is received (or attempted to be)
+var ReceiveHook func(*gomemcached.MCResponse, int, error)
+
+func getResponse(s io.Reader, hdrBytes []byte) (rv *gomemcached.MCResponse, n int, err error) {
+	if s == nil {
+		return nil, 0, errNoConn
+	}
+
+	rv = &gomemcached.MCResponse{}
+	n, err = rv.Receive(s, hdrBytes)
+
+	if ReceiveHook != nil {
+		ReceiveHook(rv, n, err)
+	}
+
+	if err == nil && (rv.Status != gomemcached.SUCCESS && rv.Status != gomemcached.AUTH_CONTINUE) {
+		err = rv
+	}
+	return rv, n, err
+}
+
+// TransmitHook is called after each packet is transmitted.
+var TransmitHook func(*gomemcached.MCRequest, int, error)
+
+func transmitRequest(o io.Writer, req *gomemcached.MCRequest) (int, error) {
+	if o == nil {
+		return 0, errNoConn
+	}
+	n, err := req.Transmit(o)
+	if TransmitHook != nil {
+		TransmitHook(req, n, err)
+	}
+	return n, err
+}
+
+func transmitResponse(o io.Writer, res *gomemcached.MCResponse) (int, error) {
+	if o == nil {
+		return 0, errNoConn
+	}
+	n, err := res.Transmit(o)
+	return n, err
+}

vendor/github.com/couchbase/gomemcached/mc_constants.go

@@ -0,0 +1,335 @@
+// Package gomemcached is binary protocol packet formats and constants.
+package gomemcached
+
+import (
+	"fmt"
+)
+
+const (
+	REQ_MAGIC = 0x80
+	RES_MAGIC = 0x81
+)
+
+// CommandCode for memcached packets.
+type CommandCode uint8
+
+const (
+	GET        = CommandCode(0x00)
+	SET        = CommandCode(0x01)
+	ADD        = CommandCode(0x02)
+	REPLACE    = CommandCode(0x03)
+	DELETE     = CommandCode(0x04)
+	INCREMENT  = CommandCode(0x05)
+	DECREMENT  = CommandCode(0x06)
+	QUIT       = CommandCode(0x07)
+	FLUSH      = CommandCode(0x08)
+	GETQ       = CommandCode(0x09)
+	NOOP       = CommandCode(0x0a)
+	VERSION    = CommandCode(0x0b)
+	GETK       = CommandCode(0x0c)
+	GETKQ      = CommandCode(0x0d)
+	APPEND     = CommandCode(0x0e)
+	PREPEND    = CommandCode(0x0f)
+	STAT       = CommandCode(0x10)
+	SETQ       = CommandCode(0x11)
+	ADDQ       = CommandCode(0x12)
+	REPLACEQ   = CommandCode(0x13)
+	DELETEQ    = CommandCode(0x14)
+	INCREMENTQ = CommandCode(0x15)
+	DECREMENTQ = CommandCode(0x16)
+	QUITQ      = CommandCode(0x17)
+	FLUSHQ     = CommandCode(0x18)
+	APPENDQ    = CommandCode(0x19)
+	AUDIT      = CommandCode(0x27)
+	PREPENDQ   = CommandCode(0x1a)
+	GAT        = CommandCode(0x1d)
+	HELLO      = CommandCode(0x1f)
+	RGET       = CommandCode(0x30)
+	RSET       = CommandCode(0x31)
+	RSETQ      = CommandCode(0x32)
+	RAPPEND    = CommandCode(0x33)
+	RAPPENDQ   = CommandCode(0x34)
+	RPREPEND   = CommandCode(0x35)
+	RPREPENDQ  = CommandCode(0x36)
+	RDELETE    = CommandCode(0x37)
+	RDELETEQ   = CommandCode(0x38)
+	RINCR      = CommandCode(0x39)
+	RINCRQ     = CommandCode(0x3a)
+	RDECR      = CommandCode(0x3b)
+	RDECRQ     = CommandCode(0x3c)
+
+	SASL_LIST_MECHS = CommandCode(0x20)
+	SASL_AUTH       = CommandCode(0x21)
+	SASL_STEP       = CommandCode(0x22)
+
+	SET_VBUCKET = CommandCode(0x3d)
+
+	TAP_CONNECT          = CommandCode(0x40) // Client-sent request to initiate Tap feed
+	TAP_MUTATION         = CommandCode(0x41) // Notification of a SET/ADD/REPLACE/etc. on the server
+	TAP_DELETE           = CommandCode(0x42) // Notification of a DELETE on the server
+	TAP_FLUSH            = CommandCode(0x43) // Replicates a flush_all command
+	TAP_OPAQUE           = CommandCode(0x44) // Opaque control data from the engine
+	TAP_VBUCKET_SET      = CommandCode(0x45) // Sets state of vbucket in receiver (used in takeover)
+	TAP_CHECKPOINT_START = CommandCode(0x46) // Notifies start of new checkpoint
+	TAP_CHECKPOINT_END   = CommandCode(0x47) // Notifies end of checkpoint
+
+	UPR_OPEN        = CommandCode(0x50) // Open a UPR connection with a name
+	UPR_ADDSTREAM   = CommandCode(0x51) // Sent by ebucketMigrator to UPR Consumer
+	UPR_CLOSESTREAM = CommandCode(0x52) // Sent by eBucketMigrator to UPR Consumer
+	UPR_FAILOVERLOG = CommandCode(0x54) // Request failover logs
+	UPR_STREAMREQ   = CommandCode(0x53) // Stream request from consumer to producer
+	UPR_STREAMEND   = CommandCode(0x55) // Sent by producer when it has no more messages to stream
+	UPR_SNAPSHOT    = CommandCode(0x56) // Start of a new snapshot
+	UPR_MUTATION    = CommandCode(0x57) // Key mutation
+	UPR_DELETION    = CommandCode(0x58) // Key deletion
+	UPR_EXPIRATION  = CommandCode(0x59) // Key expiration
+	UPR_FLUSH       = CommandCode(0x5a) // Delete all the data for a vbucket
+	UPR_NOOP        = CommandCode(0x5c) // UPR NOOP
+	UPR_BUFFERACK   = CommandCode(0x5d) // UPR Buffer Acknowledgement
+	UPR_CONTROL     = CommandCode(0x5e) // Set flow control params
+
+	SELECT_BUCKET = CommandCode(0x89) // Select bucket
+
+	OBSERVE_SEQNO = CommandCode(0x91) // Sequence Number based Observe
+	OBSERVE       = CommandCode(0x92)
+
+	GET_META            = CommandCode(0xA0) // Get meta. returns with expiry, flags, cas etc
+	SUBDOC_GET          = CommandCode(0xc5) // Get subdoc. Returns with xattrs
+	SUBDOC_MULTI_LOOKUP = CommandCode(0xd0) // Multi lookup. Doc xattrs and meta.
+)
+
+// command codes that are counted toward DCP control buffer
+// when DCP clients receive DCP messages with these command codes, they need to provide acknowledgement
+var BufferedCommandCodeMap = map[CommandCode]bool{
+	SET_VBUCKET:    true,
+	UPR_STREAMEND:  true,
+	UPR_SNAPSHOT:   true,
+	UPR_MUTATION:   true,
+	UPR_DELETION:   true,
+	UPR_EXPIRATION: true}
+
+// Status field for memcached response.
+type Status uint16
+
+// Matches with protocol_binary.h as source of truth
+const (
+	SUCCESS         = Status(0x00)
+	KEY_ENOENT      = Status(0x01)
+	KEY_EEXISTS     = Status(0x02)
+	E2BIG           = Status(0x03)
+	EINVAL          = Status(0x04)
+	NOT_STORED      = Status(0x05)
+	DELTA_BADVAL    = Status(0x06)
+	NOT_MY_VBUCKET  = Status(0x07)
+	NO_BUCKET       = Status(0x08)
+	LOCKED          = Status(0x09)
+	AUTH_STALE      = Status(0x1f)
+	AUTH_ERROR      = Status(0x20)
+	AUTH_CONTINUE   = Status(0x21)
+	ERANGE          = Status(0x22)
+	ROLLBACK        = Status(0x23)
+	EACCESS         = Status(0x24)
+	NOT_INITIALIZED = Status(0x25)
+	UNKNOWN_COMMAND = Status(0x81)
+	ENOMEM          = Status(0x82)
+	NOT_SUPPORTED   = Status(0x83)
+	EINTERNAL       = Status(0x84)
+	EBUSY           = Status(0x85)
+	TMPFAIL         = Status(0x86)
+
+	// SUBDOC
+	SUBDOC_PATH_NOT_FOUND             = Status(0xc0)
+	SUBDOC_BAD_MULTI                  = Status(0xcc)
+	SUBDOC_MULTI_PATH_FAILURE_DELETED = Status(0xd3)
+)
+
+// for log redaction
+const (
+	UdTagBegin = "<ud>"
+	UdTagEnd   = "</ud>"
+)
+
+var isFatal = map[Status]bool{
+	DELTA_BADVAL:  true,
+	NO_BUCKET:     true,
+	AUTH_STALE:    true,
+	AUTH_ERROR:    true,
+	ERANGE:        true,
+	ROLLBACK:      true,
+	EACCESS:       true,
+	ENOMEM:        true,
+	NOT_SUPPORTED: true,
+}
+
+// the producer/consumer bit in dcp flags
+var DCP_PRODUCER uint32 = 0x01
+
+// the include XATTRS bit in dcp flags
+var DCP_OPEN_INCLUDE_XATTRS uint32 = 0x04
+
+// the include deletion time bit in dcp flags
+var DCP_OPEN_INCLUDE_DELETE_TIMES uint32 = 0x20
+
+// Datatype to Include XATTRS in SUBDOC GET
+var SUBDOC_FLAG_XATTR uint8 = 0x04
+
+// MCItem is an internal representation of an item.
+type MCItem struct {
+	Cas               uint64
+	Flags, Expiration uint32
+	Data              []byte
+}
+
+// Number of bytes in a binary protocol header.
+const HDR_LEN = 24
+
+// Mapping of CommandCode -> name of command (not exhaustive)
+var CommandNames map[CommandCode]string
+
+// StatusNames human readable names for memcached response.
+var StatusNames map[Status]string
+
+func init() {
+	CommandNames = make(map[CommandCode]string)
+	CommandNames[GET] = "GET"
+	CommandNames[SET] = "SET"
+	CommandNames[ADD] = "ADD"
+	CommandNames[REPLACE] = "REPLACE"
+	CommandNames[DELETE] = "DELETE"
+	CommandNames[INCREMENT] = "INCREMENT"
+	CommandNames[DECREMENT] = "DECREMENT"
+	CommandNames[QUIT] = "QUIT"
+	CommandNames[FLUSH] = "FLUSH"
+	CommandNames[GETQ] = "GETQ"
+	CommandNames[NOOP] = "NOOP"
+	CommandNames[VERSION] = "VERSION"
+	CommandNames[GETK] = "GETK"
+	CommandNames[GETKQ] = "GETKQ"
+	CommandNames[APPEND] = "APPEND"
+	CommandNames[PREPEND] = "PREPEND"
+	CommandNames[STAT] = "STAT"
+	CommandNames[SETQ] = "SETQ"
+	CommandNames[ADDQ] = "ADDQ"
+	CommandNames[REPLACEQ] = "REPLACEQ"
+	CommandNames[DELETEQ] = "DELETEQ"
+	CommandNames[INCREMENTQ] = "INCREMENTQ"
+	CommandNames[DECREMENTQ] = "DECREMENTQ"
+	CommandNames[QUITQ] = "QUITQ"
+	CommandNames[FLUSHQ] = "FLUSHQ"
+	CommandNames[APPENDQ] = "APPENDQ"
+	CommandNames[PREPENDQ] = "PREPENDQ"
+	CommandNames[RGET] = "RGET"
+	CommandNames[RSET] = "RSET"
+	CommandNames[RSETQ] = "RSETQ"
+	CommandNames[RAPPEND] = "RAPPEND"
+	CommandNames[RAPPENDQ] = "RAPPENDQ"
+	CommandNames[RPREPEND] = "RPREPEND"
+	CommandNames[RPREPENDQ] = "RPREPENDQ"
+	CommandNames[RDELETE] = "RDELETE"
+	CommandNames[RDELETEQ] = "RDELETEQ"
+	CommandNames[RINCR] = "RINCR"
+	CommandNames[RINCRQ] = "RINCRQ"
+	CommandNames[RDECR] = "RDECR"
+	CommandNames[RDECRQ] = "RDECRQ"
+
+	CommandNames[SASL_LIST_MECHS] = "SASL_LIST_MECHS"
+	CommandNames[SASL_AUTH] = "SASL_AUTH"
+	CommandNames[SASL_STEP] = "SASL_STEP"
+
+	CommandNames[TAP_CONNECT] = "TAP_CONNECT"
+	CommandNames[TAP_MUTATION] = "TAP_MUTATION"
+	CommandNames[TAP_DELETE] = "TAP_DELETE"
+	CommandNames[TAP_FLUSH] = "TAP_FLUSH"
+	CommandNames[TAP_OPAQUE] = "TAP_OPAQUE"
+	CommandNames[TAP_VBUCKET_SET] = "TAP_VBUCKET_SET"
+	CommandNames[TAP_CHECKPOINT_START] = "TAP_CHECKPOINT_START"
+	CommandNames[TAP_CHECKPOINT_END] = "TAP_CHECKPOINT_END"
+
+	CommandNames[UPR_OPEN] = "UPR_OPEN"
+	CommandNames[UPR_ADDSTREAM] = "UPR_ADDSTREAM"
+	CommandNames[UPR_CLOSESTREAM] = "UPR_CLOSESTREAM"
+	CommandNames[UPR_FAILOVERLOG] = "UPR_FAILOVERLOG"
+	CommandNames[UPR_STREAMREQ] = "UPR_STREAMREQ"
+	CommandNames[UPR_STREAMEND] = "UPR_STREAMEND"
+	CommandNames[UPR_SNAPSHOT] = "UPR_SNAPSHOT"
+	CommandNames[UPR_MUTATION] = "UPR_MUTATION"
+	CommandNames[UPR_DELETION] = "UPR_DELETION"
+	CommandNames[UPR_EXPIRATION] = "UPR_EXPIRATION"
+	CommandNames[UPR_FLUSH] = "UPR_FLUSH"
+	CommandNames[UPR_NOOP] = "UPR_NOOP"
+	CommandNames[UPR_BUFFERACK] = "UPR_BUFFERACK"
+	CommandNames[UPR_CONTROL] = "UPR_CONTROL"
+	CommandNames[SUBDOC_GET] = "SUBDOC_GET"
+	CommandNames[SUBDOC_MULTI_LOOKUP] = "SUBDOC_MULTI_LOOKUP"
+
+	StatusNames = make(map[Status]string)
+	StatusNames[SUCCESS] = "SUCCESS"
+	StatusNames[KEY_ENOENT] = "KEY_ENOENT"
+	StatusNames[KEY_EEXISTS] = "KEY_EEXISTS"
+	StatusNames[E2BIG] = "E2BIG"
+	StatusNames[EINVAL] = "EINVAL"
+	StatusNames[NOT_STORED] = "NOT_STORED"
+	StatusNames[DELTA_BADVAL] = "DELTA_BADVAL"
+	StatusNames[NOT_MY_VBUCKET] = "NOT_MY_VBUCKET"
+	StatusNames[NO_BUCKET] = "NO_BUCKET"
+	StatusNames[AUTH_STALE] = "AUTH_STALE"
+	StatusNames[AUTH_ERROR] = "AUTH_ERROR"
+	StatusNames[AUTH_CONTINUE] = "AUTH_CONTINUE"
+	StatusNames[ERANGE] = "ERANGE"
+	StatusNames[ROLLBACK] = "ROLLBACK"
+	StatusNames[EACCESS] = "EACCESS"
+	StatusNames[NOT_INITIALIZED] = "NOT_INITIALIZED"
+	StatusNames[UNKNOWN_COMMAND] = "UNKNOWN_COMMAND"
+	StatusNames[ENOMEM] = "ENOMEM"
+	StatusNames[NOT_SUPPORTED] = "NOT_SUPPORTED"
+	StatusNames[EINTERNAL] = "EINTERNAL"
+	StatusNames[EBUSY] = "EBUSY"
+	StatusNames[TMPFAIL] = "TMPFAIL"
+	StatusNames[SUBDOC_PATH_NOT_FOUND] = "SUBDOC_PATH_NOT_FOUND"
+	StatusNames[SUBDOC_BAD_MULTI] = "SUBDOC_BAD_MULTI"
+
+}
+
+// String an op code.
+func (o CommandCode) String() (rv string) {
+	rv = CommandNames[o]
+	if rv == "" {
+		rv = fmt.Sprintf("0x%02x", int(o))
+	}
+	return rv
+}
+
+// String an op code.
+func (s Status) String() (rv string) {
+	rv = StatusNames[s]
+	if rv == "" {
+		rv = fmt.Sprintf("0x%02x", int(s))
+	}
+	return rv
+}
+
+// IsQuiet will return true if a command is a "quiet" command.
+func (o CommandCode) IsQuiet() bool {
+	switch o {
+	case GETQ,
+		GETKQ,
+		SETQ,
+		ADDQ,
+		REPLACEQ,
+		DELETEQ,
+		INCREMENTQ,
+		DECREMENTQ,
+		QUITQ,
+		FLUSHQ,
+		APPENDQ,
+		PREPENDQ,
+		RSETQ,
+		RAPPENDQ,
+		RPREPENDQ,
+		RDELETEQ,
+		RINCRQ,
+		RDECRQ:
+		return true
+	}
+	return false
+}

vendor/github.com/couchbase/gomemcached/mc_req.go

@@ -0,0 +1,197 @@
+package gomemcached
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+)
+
+// The maximum reasonable body length to expect.
+// Anything larger than this will result in an error.
+// The current limit, 20MB, is the size limit supported by ep-engine.
+var MaxBodyLen = int(20 * 1024 * 1024)
+
+// MCRequest is memcached Request
+type MCRequest struct {
+	// The command being issued
+	Opcode CommandCode
+	// The CAS (if applicable, or 0)
+	Cas uint64
+	// An opaque value to be returned with this request
+	Opaque uint32
+	// The vbucket to which this command belongs
+	VBucket uint16
+	// Command extras, key, and body
+	Extras, Key, Body, ExtMeta []byte
+	// Datatype identifier
+	DataType uint8
+}
+
+// Size gives the number of bytes this request requires.
+func (req *MCRequest) Size() int {
+	return HDR_LEN + len(req.Extras) + len(req.Key) + len(req.Body) + len(req.ExtMeta)
+}
+
+// A debugging string representation of this request
+func (req MCRequest) String() string {
+	return fmt.Sprintf("{MCRequest opcode=%s, bodylen=%d, key='%s'}",
+		req.Opcode, len(req.Body), req.Key)
+}
+
+func (req *MCRequest) fillHeaderBytes(data []byte) int {
+
+	pos := 0
+	data[pos] = REQ_MAGIC
+	pos++
+	data[pos] = byte(req.Opcode)
+	pos++
+	binary.BigEndian.PutUint16(data[pos:pos+2],
+		uint16(len(req.Key)))
+	pos += 2
+
+	// 4
+	data[pos] = byte(len(req.Extras))
+	pos++
+	// Data type
+	if req.DataType != 0 {
+		data[pos] = byte(req.DataType)
+	}
+	pos++
+	binary.BigEndian.PutUint16(data[pos:pos+2], req.VBucket)
+	pos += 2
+
+	// 8
+	binary.BigEndian.PutUint32(data[pos:pos+4],
+		uint32(len(req.Body)+len(req.Key)+len(req.Extras)+len(req.ExtMeta)))
+	pos += 4
+
+	// 12
+	binary.BigEndian.PutUint32(data[pos:pos+4], req.Opaque)
+	pos += 4
+
+	// 16
+	if req.Cas != 0 {
+		binary.BigEndian.PutUint64(data[pos:pos+8], req.Cas)
+	}
+	pos += 8
+
+	if len(req.Extras) > 0 {
+		copy(data[pos:pos+len(req.Extras)], req.Extras)
+		pos += len(req.Extras)
+	}
+
+	if len(req.Key) > 0 {
+		copy(data[pos:pos+len(req.Key)], req.Key)
+		pos += len(req.Key)
+	}
+
+	return pos
+}
+
+// HeaderBytes will return the wire representation of the request header
+// (with the extras and key).
+func (req *MCRequest) HeaderBytes() []byte {
+	data := make([]byte, HDR_LEN+len(req.Extras)+len(req.Key))
+
+	req.fillHeaderBytes(data)
+
+	return data
+}
+
+// Bytes will return the wire representation of this request.
+func (req *MCRequest) Bytes() []byte {
+	data := make([]byte, req.Size())
+
+	pos := req.fillHeaderBytes(data)
+
+	if len(req.Body) > 0 {
+		copy(data[pos:pos+len(req.Body)], req.Body)
+	}
+
+	if len(req.ExtMeta) > 0 {
+		copy(data[pos+len(req.Body):pos+len(req.Body)+len(req.ExtMeta)], req.ExtMeta)
+	}
+
+	return data
+}
+
+// Transmit will send this request message across a writer.
+func (req *MCRequest) Transmit(w io.Writer) (n int, err error) {
+	if len(req.Body) < 128 {
+		n, err = w.Write(req.Bytes())
+	} else {
+		n, err = w.Write(req.HeaderBytes())
+		if err == nil {
+			m := 0
+			m, err = w.Write(req.Body)
+			n += m
+		}
+	}
+	return
+}
+
+// Receive will fill this MCRequest with the data from a reader.
+func (req *MCRequest) Receive(r io.Reader, hdrBytes []byte) (int, error) {
+	if len(hdrBytes) < HDR_LEN {
+		hdrBytes = []byte{
+			0, 0, 0, 0, 0, 0, 0, 0,
+			0, 0, 0, 0, 0, 0, 0, 0,
+			0, 0, 0, 0, 0, 0, 0, 0}
+	}
+	n, err := io.ReadFull(r, hdrBytes)
+	if err != nil {
+		return n, err
+	}
+
+	if hdrBytes[0] != RES_MAGIC && hdrBytes[0] != REQ_MAGIC {
+		return n, fmt.Errorf("bad magic: 0x%02x", hdrBytes[0])
+	}
+
+	klen := int(binary.BigEndian.Uint16(hdrBytes[2:]))
+	elen := int(hdrBytes[4])
+	// Data type at 5
+	req.DataType = uint8(hdrBytes[5])
+
+	req.Opcode = CommandCode(hdrBytes[1])
+	// Vbucket at 6:7
+	req.VBucket = binary.BigEndian.Uint16(hdrBytes[6:])
+	totalBodyLen := int(binary.BigEndian.Uint32(hdrBytes[8:]))
+
+	req.Opaque = binary.BigEndian.Uint32(hdrBytes[12:])
+	req.Cas = binary.BigEndian.Uint64(hdrBytes[16:])
+
+	if totalBodyLen > 0 {
+		buf := make([]byte, totalBodyLen)
+		m, err := io.ReadFull(r, buf)
+		n += m
+		if err == nil {
+			if req.Opcode >= TAP_MUTATION &&
+				req.Opcode <= TAP_CHECKPOINT_END &&
+				len(buf) > 1 {
+				// In these commands there is "engine private"
+				// data at the end of the extras.  The first 2
+				// bytes of extra data give its length.
+				elen += int(binary.BigEndian.Uint16(buf))
+			}
+
+			req.Extras = buf[0:elen]
+			req.Key = buf[elen : klen+elen]
+
+			// get the length of extended metadata
+			extMetaLen := 0
+			if elen > 29 {
+				extMetaLen = int(binary.BigEndian.Uint16(req.Extras[28:30]))
+			}
+
+			bodyLen := totalBodyLen - klen - elen - extMetaLen
+			if bodyLen > MaxBodyLen {
+				return n, fmt.Errorf("%d is too big (max %d)",
+					bodyLen, MaxBodyLen)
+			}
+
+			req.Body = buf[klen+elen : klen+elen+bodyLen]
+			req.ExtMeta = buf[klen+elen+bodyLen:]
+		}
+	}
+	return n, err
+}

vendor/github.com/couchbase/gomemcached/mc_res.go

@@ -0,0 +1,267 @@
+package gomemcached
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"sync"
+)
+
+// MCResponse is memcached response
+type MCResponse struct {
+	// The command opcode of the command that sent the request
+	Opcode CommandCode
+	// The status of the response
+	Status Status
+	// The opaque sent in the request
+	Opaque uint32
+	// The CAS identifier (if applicable)
+	Cas uint64
+	// Extras, key, and body for this response
+	Extras, Key, Body []byte
+	// If true, this represents a fatal condition and we should hang up
+	Fatal bool
+	// Datatype identifier
+	DataType uint8
+}
+
+// A debugging string representation of this response
+func (res MCResponse) String() string {
+	return fmt.Sprintf("{MCResponse status=%v keylen=%d, extralen=%d, bodylen=%d}",
+		res.Status, len(res.Key), len(res.Extras), len(res.Body))
+}
+
+// Response as an error.
+func (res *MCResponse) Error() string {
+	return fmt.Sprintf("MCResponse status=%v, opcode=%v, opaque=%v, msg: %s",
+		res.Status, res.Opcode, res.Opaque, string(res.Body))
+}
+
+func errStatus(e error) Status {
+	status := Status(0xffff)
+	if res, ok := e.(*MCResponse); ok {
+		status = res.Status
+	}
+	return status
+}
+
+// IsNotFound is true if this error represents a "not found" response.
+func IsNotFound(e error) bool {
+	return errStatus(e) == KEY_ENOENT
+}
+
+// IsFatal is false if this error isn't believed to be fatal to a connection.
+func IsFatal(e error) bool {
+	if e == nil {
+		return false
+	}
+	_, ok := isFatal[errStatus(e)]
+	if ok {
+		return true
+	}
+	return false
+}
+
+// Size is number of bytes this response consumes on the wire.
+func (res *MCResponse) Size() int {
+	return HDR_LEN + len(res.Extras) + len(res.Key) + len(res.Body)
+}
+
+func (res *MCResponse) fillHeaderBytes(data []byte) int {
+	pos := 0
+	data[pos] = RES_MAGIC
+	pos++
+	data[pos] = byte(res.Opcode)
+	pos++
+	binary.BigEndian.PutUint16(data[pos:pos+2],
+		uint16(len(res.Key)))
+	pos += 2
+
+	// 4
+	data[pos] = byte(len(res.Extras))
+	pos++
+	// Data type
+	if res.DataType != 0 {
+		data[pos] = byte(res.DataType)
+	} else {
+		data[pos] = 0
+	}
+	pos++
+	binary.BigEndian.PutUint16(data[pos:pos+2], uint16(res.Status))
+	pos += 2
+
+	// 8
+	binary.BigEndian.PutUint32(data[pos:pos+4],
+		uint32(len(res.Body)+len(res.Key)+len(res.Extras)))
+	pos += 4
+
+	// 12
+	binary.BigEndian.PutUint32(data[pos:pos+4], res.Opaque)
+	pos += 4
+
+	// 16
+	binary.BigEndian.PutUint64(data[pos:pos+8], res.Cas)
+	pos += 8
+
+	if len(res.Extras) > 0 {
+		copy(data[pos:pos+len(res.Extras)], res.Extras)
+		pos += len(res.Extras)
+	}
+
+	if len(res.Key) > 0 {
+		copy(data[pos:pos+len(res.Key)], res.Key)
+		pos += len(res.Key)
+	}
+
+	return pos
+}
+
+// HeaderBytes will get just the header bytes for this response.
+func (res *MCResponse) HeaderBytes() []byte {
+	data := make([]byte, HDR_LEN+len(res.Extras)+len(res.Key))
+
+	res.fillHeaderBytes(data)
+
+	return data
+}
+
+// Bytes will return the actual bytes transmitted for this response.
+func (res *MCResponse) Bytes() []byte {
+	data := make([]byte, res.Size())
+
+	pos := res.fillHeaderBytes(data)
+
+	copy(data[pos:pos+len(res.Body)], res.Body)
+
+	return data
+}
+
+// Transmit will send this response message across a writer.
+func (res *MCResponse) Transmit(w io.Writer) (n int, err error) {
+	if len(res.Body) < 128 {
+		n, err = w.Write(res.Bytes())
+	} else {
+		n, err = w.Write(res.HeaderBytes())
+		if err == nil {
+			m := 0
+			m, err = w.Write(res.Body)
+			m += n
+		}
+	}
+	return
+}
+
+// Receive will fill this MCResponse with the data from this reader.
+func (res *MCResponse) Receive(r io.Reader, hdrBytes []byte) (n int, err error) {
+	if len(hdrBytes) < HDR_LEN {
+		hdrBytes = []byte{
+			0, 0, 0, 0, 0, 0, 0, 0,
+			0, 0, 0, 0, 0, 0, 0, 0,
+			0, 0, 0, 0, 0, 0, 0, 0}
+	}
+	n, err = io.ReadFull(r, hdrBytes)
+	if err != nil {
+		return n, err
+	}
+
+	if hdrBytes[0] != RES_MAGIC && hdrBytes[0] != REQ_MAGIC {
+		return n, fmt.Errorf("bad magic: 0x%02x", hdrBytes[0])
+	}
+
+	klen := int(binary.BigEndian.Uint16(hdrBytes[2:4]))
+	elen := int(hdrBytes[4])
+
+	res.Opcode = CommandCode(hdrBytes[1])
+	res.DataType = uint8(hdrBytes[5])
+	res.Status = Status(binary.BigEndian.Uint16(hdrBytes[6:8]))
+	res.Opaque = binary.BigEndian.Uint32(hdrBytes[12:16])
+	res.Cas = binary.BigEndian.Uint64(hdrBytes[16:24])
+
+	bodyLen := int(binary.BigEndian.Uint32(hdrBytes[8:12])) - (klen + elen)
+
+	//defer function to debug the panic seen with MB-15557
+	defer func() {
+		if e := recover(); e != nil {
+			err = fmt.Errorf(`Panic in Receive. Response %v \n
+                        key len %v extra len %v bodylen %v`, res, klen, elen, bodyLen)
+		}
+	}()
+
+	buf := make([]byte, klen+elen+bodyLen)
+	m, err := io.ReadFull(r, buf)
+	if err == nil {
+		res.Extras = buf[0:elen]
+		res.Key = buf[elen : klen+elen]
+		res.Body = buf[klen+elen:]
+	}
+
+	return n + m, err
+}
+
+type MCResponsePool struct {
+	pool *sync.Pool
+}
+
+func NewMCResponsePool() *MCResponsePool {
+	rv := &MCResponsePool{
+		pool: &sync.Pool{
+			New: func() interface{} {
+				return &MCResponse{}
+			},
+		},
+	}
+
+	return rv
+}
+
+func (this *MCResponsePool) Get() *MCResponse {
+	return this.pool.Get().(*MCResponse)
+}
+
+func (this *MCResponsePool) Put(r *MCResponse) {
+	if r == nil {
+		return
+	}
+
+	r.Extras = nil
+	r.Key = nil
+	r.Body = nil
+	r.Fatal = false
+
+	this.pool.Put(r)
+}
+
+type StringMCResponsePool struct {
+	pool *sync.Pool
+	size int
+}
+
+func NewStringMCResponsePool(size int) *StringMCResponsePool {
+	rv := &StringMCResponsePool{
+		pool: &sync.Pool{
+			New: func() interface{} {
+				return make(map[string]*MCResponse, size)
+			},
+		},
+		size: size,
+	}
+
+	return rv
+}
+
+func (this *StringMCResponsePool) Get() map[string]*MCResponse {
+	return this.pool.Get().(map[string]*MCResponse)
+}
+
+func (this *StringMCResponsePool) Put(m map[string]*MCResponse) {
+	if m == nil || len(m) > 2*this.size {
+		return
+	}
+
+	for k := range m {
+		m[k] = nil
+		delete(m, k)
+	}
+
+	this.pool.Put(m)
+}

vendor/github.com/couchbase/gomemcached/tap.go

@@ -0,0 +1,168 @@
+package gomemcached
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+)
+
+type TapConnectFlag uint32
+
+// Tap connect option flags
+const (
+	BACKFILL           = TapConnectFlag(0x01)
+	DUMP               = TapConnectFlag(0x02)
+	LIST_VBUCKETS      = TapConnectFlag(0x04)
+	TAKEOVER_VBUCKETS  = TapConnectFlag(0x08)
+	SUPPORT_ACK        = TapConnectFlag(0x10)
+	REQUEST_KEYS_ONLY  = TapConnectFlag(0x20)
+	CHECKPOINT         = TapConnectFlag(0x40)
+	REGISTERED_CLIENT  = TapConnectFlag(0x80)
+	FIX_FLAG_BYTEORDER = TapConnectFlag(0x100)
+)
+
+// Tap opaque event subtypes
+const (
+	TAP_OPAQUE_ENABLE_AUTO_NACK       = 0
+	TAP_OPAQUE_INITIAL_VBUCKET_STREAM = 1
+	TAP_OPAQUE_ENABLE_CHECKPOINT_SYNC = 2
+	TAP_OPAQUE_CLOSE_TAP_STREAM       = 7
+	TAP_OPAQUE_CLOSE_BACKFILL         = 8
+)
+
+// Tap item flags
+const (
+	TAP_ACK                     = 1
+	TAP_NO_VALUE                = 2
+	TAP_FLAG_NETWORK_BYTE_ORDER = 4
+)
+
+// TapConnectFlagNames for TapConnectFlag
+var TapConnectFlagNames = map[TapConnectFlag]string{
+	BACKFILL:           "BACKFILL",
+	DUMP:               "DUMP",
+	LIST_VBUCKETS:      "LIST_VBUCKETS",
+	TAKEOVER_VBUCKETS:  "TAKEOVER_VBUCKETS",
+	SUPPORT_ACK:        "SUPPORT_ACK",
+	REQUEST_KEYS_ONLY:  "REQUEST_KEYS_ONLY",
+	CHECKPOINT:         "CHECKPOINT",
+	REGISTERED_CLIENT:  "REGISTERED_CLIENT",
+	FIX_FLAG_BYTEORDER: "FIX_FLAG_BYTEORDER",
+}
+
+// TapItemParser is a function to parse a single tap extra.
+type TapItemParser func(io.Reader) (interface{}, error)
+
+// TapParseUint64 is a function to parse a single tap uint64.
+func TapParseUint64(r io.Reader) (interface{}, error) {
+	var rv uint64
+	err := binary.Read(r, binary.BigEndian, &rv)
+	return rv, err
+}
+
+// TapParseUint16 is a function to parse a single tap uint16.
+func TapParseUint16(r io.Reader) (interface{}, error) {
+	var rv uint16
+	err := binary.Read(r, binary.BigEndian, &rv)
+	return rv, err
+}
+
+// TapParseBool is a function to parse a single tap boolean.
+func TapParseBool(r io.Reader) (interface{}, error) {
+	return true, nil
+}
+
+// TapParseVBList parses a list of vBucket numbers as []uint16.
+func TapParseVBList(r io.Reader) (interface{}, error) {
+	num, err := TapParseUint16(r)
+	if err != nil {
+		return nil, err
+	}
+	n := int(num.(uint16))
+
+	rv := make([]uint16, n)
+	for i := 0; i < n; i++ {
+		x, err := TapParseUint16(r)
+		if err != nil {
+			return nil, err
+		}
+		rv[i] = x.(uint16)
+	}
+
+	return rv, err
+}
+
+// TapFlagParsers parser functions for TAP fields.
+var TapFlagParsers = map[TapConnectFlag]TapItemParser{
+	BACKFILL:      TapParseUint64,
+	LIST_VBUCKETS: TapParseVBList,
+}
+
+// SplitFlags will split the ORed flags into the individual bit flags.
+func (f TapConnectFlag) SplitFlags() []TapConnectFlag {
+	rv := []TapConnectFlag{}
+	for i := uint32(1); f != 0; i = i << 1 {
+		if uint32(f)&i == i {
+			rv = append(rv, TapConnectFlag(i))
+		}
+		f = TapConnectFlag(uint32(f) & (^i))
+	}
+	return rv
+}
+
+func (f TapConnectFlag) String() string {
+	parts := []string{}
+	for _, x := range f.SplitFlags() {
+		p := TapConnectFlagNames[x]
+		if p == "" {
+			p = fmt.Sprintf("0x%x", int(x))
+		}
+		parts = append(parts, p)
+	}
+	return strings.Join(parts, "|")
+}
+
+type TapConnect struct {
+	Flags         map[TapConnectFlag]interface{}
+	RemainingBody []byte
+	Name          string
+}
+
+// ParseTapCommands parse the tap request into the interesting bits we may
+// need to do something with.
+func (req *MCRequest) ParseTapCommands() (TapConnect, error) {
+	rv := TapConnect{
+		Flags: map[TapConnectFlag]interface{}{},
+		Name:  string(req.Key),
+	}
+
+	if len(req.Extras) < 4 {
+		return rv, fmt.Errorf("not enough extra bytes: %x", req.Extras)
+	}
+
+	flags := TapConnectFlag(binary.BigEndian.Uint32(req.Extras))
+
+	r := bytes.NewReader(req.Body)
+
+	for _, f := range flags.SplitFlags() {
+		fun := TapFlagParsers[f]
+		if fun == nil {
+			fun = TapParseBool
+		}
+
+		val, err := fun(r)
+		if err != nil {
+			return rv, err
+		}
+
+		rv.Flags[f] = val
+	}
+
+	var err error
+	rv.RemainingBody, err = ioutil.ReadAll(r)
+
+	return rv, err
+}

vendor/github.com/couchbase/goutils/LICENSE.md

@@ -0,0 +1,47 @@
+COUCHBASE INC. COMMUNITY EDITION LICENSE AGREEMENT
+
+IMPORTANT-READ CAREFULLY: BY CLICKING THE "I ACCEPT" BOX OR INSTALLING,
+DOWNLOADING OR OTHERWISE USING THIS SOFTWARE AND ANY ASSOCIATED
+DOCUMENTATION, YOU, ON BEHALF OF YOURSELF OR AS AN AUTHORIZED
+REPRESENTATIVE ON BEHALF OF AN ENTITY ("LICENSEE") AGREE TO ALL THE
+TERMS OF THIS COMMUNITY EDITION LICENSE AGREEMENT (THE "AGREEMENT")
+REGARDING YOUR USE OF THE SOFTWARE.  YOU REPRESENT AND WARRANT THAT YOU
+HAVE FULL LEGAL AUTHORITY TO BIND THE LICENSEE TO THIS AGREEMENT. IF YOU
+DO NOT AGREE WITH ALL OF THESE TERMS, DO NOT SELECT THE "I ACCEPT" BOX
+AND DO NOT INSTALL, DOWNLOAD OR OTHERWISE USE THE SOFTWARE. THE
+EFFECTIVE DATE OF THIS AGREEMENT IS THE DATE ON WHICH YOU CLICK "I
+ACCEPT" OR OTHERWISE INSTALL, DOWNLOAD OR USE THE SOFTWARE.
+
+1. License Grant. Couchbase Inc. hereby grants Licensee, free of charge,
+the non-exclusive right to use, copy, merge, publish, distribute,
+sublicense, and/or sell copies of the Software, and to permit persons to
+whom the Software is furnished to do so, subject to Licensee including
+the following copyright notice in all copies or substantial portions of
+the Software:
+
+Couchbase (r) http://www.Couchbase.com Copyright 2016 Couchbase, Inc.
+
+As used in this Agreement, "Software" means the object code version of
+the applicable elastic data management server software provided by
+Couchbase Inc.
+
+2. Restrictions. Licensee will not reverse engineer, disassemble, or
+decompile the Software (except to the extent such restrictions are
+prohibited by law).
+
+3. Support. Couchbase, Inc. will provide Licensee with access to, and
+use of, the Couchbase, Inc. support forum available at the following
+URL: http://www.couchbase.org/forums/. Couchbase, Inc. may, at its
+discretion, modify, suspend or terminate support at any time upon notice
+to Licensee.
+
+4. Warranty Disclaimer and Limitation of Liability. THE SOFTWARE IS
+PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+COUCHBASE INC. OR THE AUTHORS OR COPYRIGHT HOLDERS IN THE SOFTWARE BE
+LIABLE FOR ANY CLAIM, DAMAGES (IINCLUDING, WITHOUT LIMITATION, DIRECT,
+INDIRECT OR CONSEQUENTIAL DAMAGES) OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/couchbase/goutils/logging/logger.go

@@ -0,0 +1,481 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package logging
+
+import (
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+type Level int
+
+const (
+	NONE    = Level(iota) // Disable all logging
+	FATAL                 // System is in severe error state and has to abort
+	SEVERE                // System is in severe error state and cannot recover reliably
+	ERROR                 // System is in error state but can recover and continue reliably
+	WARN                  // System approaching error state, or is in a correct but undesirable state
+	INFO                  // System-level events and status, in correct states
+	REQUEST               // Request-level events, with request-specific rlevel
+	TRACE                 // Trace detailed system execution, e.g. function entry / exit
+	DEBUG                 // Debug
+)
+
+type LogEntryFormatter int
+
+const (
+	TEXTFORMATTER = LogEntryFormatter(iota)
+	JSONFORMATTER
+	KVFORMATTER
+)
+
+func (level Level) String() string {
+	return _LEVEL_NAMES[level]
+}
+
+var _LEVEL_NAMES = []string{
+	DEBUG:   "DEBUG",
+	TRACE:   "TRACE",
+	REQUEST: "REQUEST",
+	INFO:    "INFO",
+	WARN:    "WARN",
+	ERROR:   "ERROR",
+	SEVERE:  "SEVERE",
+	FATAL:   "FATAL",
+	NONE:    "NONE",
+}
+
+var _LEVEL_MAP = map[string]Level{
+	"debug":   DEBUG,
+	"trace":   TRACE,
+	"request": REQUEST,
+	"info":    INFO,
+	"warn":    WARN,
+	"error":   ERROR,
+	"severe":  SEVERE,
+	"fatal":   FATAL,
+	"none":    NONE,
+}
+
+func ParseLevel(name string) (level Level, ok bool) {
+	level, ok = _LEVEL_MAP[strings.ToLower(name)]
+	return
+}
+
+/*
+
+Pair supports logging of key-value pairs.  Keys beginning with _ are
+reserved for the logger, e.g. _time, _level, _msg, and _rlevel. The
+Pair APIs are designed to avoid heap allocation and garbage
+collection.
+
+*/
+type Pairs []Pair
+type Pair struct {
+	Name  string
+	Value interface{}
+}
+
+/*
+
+Map allows key-value pairs to be specified using map literals or data
+structures. For example:
+
+Errorm(msg, Map{...})
+
+Map incurs heap allocation and garbage collection, so the Pair APIs
+should be preferred.
+
+*/
+type Map map[string]interface{}
+
+// Logger provides a common interface for logging libraries
+type Logger interface {
+	/*
+		These APIs write all the given pairs in addition to standard logger keys.
+	*/
+	Logp(level Level, msg string, kv ...Pair)
+
+	Debugp(msg string, kv ...Pair)
+
+	Tracep(msg string, kv ...Pair)
+
+	Requestp(rlevel Level, msg string, kv ...Pair)
+
+	Infop(msg string, kv ...Pair)
+
+	Warnp(msg string, kv ...Pair)
+
+	Errorp(msg string, kv ...Pair)
+
+	Severep(msg string, kv ...Pair)
+
+	Fatalp(msg string, kv ...Pair)
+
+	/*
+		These APIs write the fields in the given kv Map in addition to standard logger keys.
+	*/
+	Logm(level Level, msg string, kv Map)
+
+	Debugm(msg string, kv Map)
+
+	Tracem(msg string, kv Map)
+
+	Requestm(rlevel Level, msg string, kv Map)
+
+	Infom(msg string, kv Map)
+
+	Warnm(msg string, kv Map)
+
+	Errorm(msg string, kv Map)
+
+	Severem(msg string, kv Map)
+
+	Fatalm(msg string, kv Map)
+
+	/*
+
+		These APIs only write _msg, _time, _level, and other logger keys. If
+		the msg contains other fields, use the Pair or Map APIs instead.
+
+	*/
+	Logf(level Level, fmt string, args ...interface{})
+
+	Debugf(fmt string, args ...interface{})
+
+	Tracef(fmt string, args ...interface{})
+
+	Requestf(rlevel Level, fmt string, args ...interface{})
+
+	Infof(fmt string, args ...interface{})
+
+	Warnf(fmt string, args ...interface{})
+
+	Errorf(fmt string, args ...interface{})
+
+	Severef(fmt string, args ...interface{})
+
+	Fatalf(fmt string, args ...interface{})
+
+	/*
+		These APIs control the logging level
+	*/
+
+	SetLevel(Level) // Set the logging level
+
+	Level() Level // Get the current logging level
+}
+
+var logger Logger = nil
+var curLevel Level = DEBUG // initially set to never skip
+
+var loggerMutex sync.RWMutex
+
+// All the methods below first acquire the mutex (mostly in exclusive mode)
+// and only then check if logging at the current level is enabled.
+// This introduces a fair bottleneck for those log entries that should be
+// skipped (the majority, at INFO or below levels)
+// We try to predict here if we should lock the mutex at all by caching
+// the current log level: while dynamically changing logger, there might
+// be the odd entry skipped as the new level is cached.
+// Since we seem to never change the logger, this is not an issue.
+func skipLogging(level Level) bool {
+	if logger == nil {
+		return true
+	}
+	return level > curLevel
+}
+
+func SetLogger(newLogger Logger) {
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger = newLogger
+	if logger == nil {
+		curLevel = NONE
+	} else {
+		curLevel = newLogger.Level()
+	}
+}
+
+func Logp(level Level, msg string, kv ...Pair) {
+	if skipLogging(level) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Logp(level, msg, kv...)
+}
+
+func Debugp(msg string, kv ...Pair) {
+	if skipLogging(DEBUG) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Debugp(msg, kv...)
+}
+
+func Tracep(msg string, kv ...Pair) {
+	if skipLogging(TRACE) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Tracep(msg, kv...)
+}
+
+func Requestp(rlevel Level, msg string, kv ...Pair) {
+	if skipLogging(REQUEST) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Requestp(rlevel, msg, kv...)
+}
+
+func Infop(msg string, kv ...Pair) {
+	if skipLogging(INFO) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Infop(msg, kv...)
+}
+
+func Warnp(msg string, kv ...Pair) {
+	if skipLogging(WARN) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Warnp(msg, kv...)
+}
+
+func Errorp(msg string, kv ...Pair) {
+	if skipLogging(ERROR) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Errorp(msg, kv...)
+}
+
+func Severep(msg string, kv ...Pair) {
+	if skipLogging(SEVERE) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Severep(msg, kv...)
+}
+
+func Fatalp(msg string, kv ...Pair) {
+	if skipLogging(FATAL) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Fatalp(msg, kv...)
+}
+
+func Logm(level Level, msg string, kv Map) {
+	if skipLogging(level) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Logm(level, msg, kv)
+}
+
+func Debugm(msg string, kv Map) {
+	if skipLogging(DEBUG) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Debugm(msg, kv)
+}
+
+func Tracem(msg string, kv Map) {
+	if skipLogging(TRACE) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Tracem(msg, kv)
+}
+
+func Requestm(rlevel Level, msg string, kv Map) {
+	if skipLogging(REQUEST) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Requestm(rlevel, msg, kv)
+}
+
+func Infom(msg string, kv Map) {
+	if skipLogging(INFO) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Infom(msg, kv)
+}
+
+func Warnm(msg string, kv Map) {
+	if skipLogging(WARN) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Warnm(msg, kv)
+}
+
+func Errorm(msg string, kv Map) {
+	if skipLogging(ERROR) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Errorm(msg, kv)
+}
+
+func Severem(msg string, kv Map) {
+	if skipLogging(SEVERE) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Severem(msg, kv)
+}
+
+func Fatalm(msg string, kv Map) {
+	if skipLogging(FATAL) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Fatalm(msg, kv)
+}
+
+func Logf(level Level, fmt string, args ...interface{}) {
+	if skipLogging(level) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Logf(level, fmt, args...)
+}
+
+func Debugf(fmt string, args ...interface{}) {
+	if skipLogging(DEBUG) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Debugf(fmt, args...)
+}
+
+func Tracef(fmt string, args ...interface{}) {
+	if skipLogging(TRACE) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Tracef(fmt, args...)
+}
+
+func Requestf(rlevel Level, fmt string, args ...interface{}) {
+	if skipLogging(REQUEST) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Requestf(rlevel, fmt, args...)
+}
+
+func Infof(fmt string, args ...interface{}) {
+	if skipLogging(INFO) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Infof(fmt, args...)
+}
+
+func Warnf(fmt string, args ...interface{}) {
+	if skipLogging(WARN) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Warnf(fmt, args...)
+}
+
+func Errorf(fmt string, args ...interface{}) {
+	if skipLogging(ERROR) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Errorf(fmt, args...)
+}
+
+func Severef(fmt string, args ...interface{}) {
+	if skipLogging(SEVERE) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Severef(fmt, args...)
+}
+
+func Fatalf(fmt string, args ...interface{}) {
+	if skipLogging(FATAL) {
+		return
+	}
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Fatalf(fmt, args...)
+}
+
+func SetLevel(level Level) {
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.SetLevel(level)
+	curLevel = level
+}
+
+func LogLevel() Level {
+	loggerMutex.RLock()
+	defer loggerMutex.RUnlock()
+	return logger.Level()
+}
+
+func Stackf(level Level, fmt string, args ...interface{}) {
+	if skipLogging(level) {
+		return
+	}
+	buf := make([]byte, 1<<16)
+	n := runtime.Stack(buf, false)
+	s := string(buf[0:n])
+	loggerMutex.Lock()
+	defer loggerMutex.Unlock()
+	logger.Logf(level, fmt, args...)
+	logger.Logf(level, s)
+}
+
+func init() {
+	logger = NewLogger(os.Stderr, INFO, TEXTFORMATTER)
+	SetLogger(logger)
+}

vendor/github.com/couchbase/goutils/logging/logger_golog.go

@@ -0,0 +1,318 @@
+//  Copyright (c) 2016 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package logging
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"time"
+)
+
+type goLogger struct {
+	logger         *log.Logger
+	level          Level
+	entryFormatter formatter
+}
+
+const (
+	_LEVEL  = "_level"
+	_MSG    = "_msg"
+	_TIME   = "_time"
+	_RLEVEL = "_rlevel"
+)
+
+func NewLogger(out io.Writer, lvl Level, fmtLogging LogEntryFormatter) *goLogger {
+	logger := &goLogger{
+		logger: log.New(out, "", 0),
+		level:  lvl,
+	}
+	if fmtLogging == JSONFORMATTER {
+		logger.entryFormatter = &jsonFormatter{}
+	} else if fmtLogging == KVFORMATTER {
+		logger.entryFormatter = &keyvalueFormatter{}
+	} else {
+		logger.entryFormatter = &textFormatter{}
+	}
+	return logger
+}
+
+func (gl *goLogger) Logp(level Level, msg string, kv ...Pair) {
+	if gl.logger == nil {
+		return
+	}
+	if level <= gl.level {
+		e := newLogEntry(msg, level)
+		copyPairs(e, kv)
+		gl.log(e)
+	}
+}
+
+func (gl *goLogger) Debugp(msg string, kv ...Pair) {
+	gl.Logp(DEBUG, msg, kv...)
+}
+
+func (gl *goLogger) Tracep(msg string, kv ...Pair) {
+	gl.Logp(TRACE, msg, kv...)
+}
+
+func (gl *goLogger) Requestp(rlevel Level, msg string, kv ...Pair) {
+	if gl.logger == nil {
+		return
+	}
+	if REQUEST <= gl.level {
+		e := newLogEntry(msg, REQUEST)
+		e.Rlevel = rlevel
+		copyPairs(e, kv)
+		gl.log(e)
+	}
+}
+
+func (gl *goLogger) Infop(msg string, kv ...Pair) {
+	gl.Logp(INFO, msg, kv...)
+}
+
+func (gl *goLogger) Warnp(msg string, kv ...Pair) {
+	gl.Logp(WARN, msg, kv...)
+}
+
+func (gl *goLogger) Errorp(msg string, kv ...Pair) {
+	gl.Logp(ERROR, msg, kv...)
+}
+
+func (gl *goLogger) Severep(msg string, kv ...Pair) {
+	gl.Logp(SEVERE, msg, kv...)
+}
+
+func (gl *goLogger) Fatalp(msg string, kv ...Pair) {
+	gl.Logp(FATAL, msg, kv...)
+}
+
+func (gl *goLogger) Logm(level Level, msg string, kv Map) {
+	if gl.logger == nil {
+		return
+	}
+	if level <= gl.level {
+		e := newLogEntry(msg, level)
+		e.Data = kv
+		gl.log(e)
+	}
+}
+
+func (gl *goLogger) Debugm(msg string, kv Map) {
+	gl.Logm(DEBUG, msg, kv)
+}
+
+func (gl *goLogger) Tracem(msg string, kv Map) {
+	gl.Logm(TRACE, msg, kv)
+}
+
+func (gl *goLogger) Requestm(rlevel Level, msg string, kv Map) {
+	if gl.logger == nil {
+		return
+	}
+	if REQUEST <= gl.level {
+		e := newLogEntry(msg, REQUEST)
+		e.Rlevel = rlevel
+		e.Data = kv
+		gl.log(e)
+	}
+}
+
+func (gl *goLogger) Infom(msg string, kv Map) {
+	gl.Logm(INFO, msg, kv)
+}
+
+func (gl *goLogger) Warnm(msg string, kv Map) {
+	gl.Logm(WARN, msg, kv)
+}
+
+func (gl *goLogger) Errorm(msg string, kv Map) {
+	gl.Logm(ERROR, msg, kv)
+}
+
+func (gl *goLogger) Severem(msg string, kv Map) {
+	gl.Logm(SEVERE, msg, kv)
+}
+
+func (gl *goLogger) Fatalm(msg string, kv Map) {
+	gl.Logm(FATAL, msg, kv)
+}
+
+func (gl *goLogger) Logf(level Level, format string, args ...interface{}) {
+	if gl.logger == nil {
+		return
+	}
+	if level <= gl.level {
+		e := newLogEntry(fmt.Sprintf(format, args...), level)
+		gl.log(e)
+	}
+}
+
+func (gl *goLogger) Debugf(format string, args ...interface{}) {
+	gl.Logf(DEBUG, format, args...)
+}
+
+func (gl *goLogger) Tracef(format string, args ...interface{}) {
+	gl.Logf(TRACE, format, args...)
+}
+
+func (gl *goLogger) Requestf(rlevel Level, format string, args ...interface{}) {
+	if gl.logger == nil {
+		return
+	}
+	if REQUEST <= gl.level {
+		e := newLogEntry(fmt.Sprintf(format, args...), REQUEST)
+		e.Rlevel = rlevel
+		gl.log(e)
+	}
+}
+
+func (gl *goLogger) Infof(format string, args ...interface{}) {
+	gl.Logf(INFO, format, args...)
+}
+
+func (gl *goLogger) Warnf(format string, args ...interface{}) {
+	gl.Logf(WARN, format, args...)
+}
+
+func (gl *goLogger) Errorf(format string, args ...interface{}) {
+	gl.Logf(ERROR, format, args...)
+}
+
+func (gl *goLogger) Severef(format string, args ...interface{}) {
+	gl.Logf(SEVERE, format, args...)
+}
+
+func (gl *goLogger) Fatalf(format string, args ...interface{}) {
+	gl.Logf(FATAL, format, args...)
+}
+
+func (gl *goLogger) Level() Level {
+	return gl.level
+}
+
+func (gl *goLogger) SetLevel(level Level) {
+	gl.level = level
+}
+
+func (gl *goLogger) log(newEntry *logEntry) {
+	s := gl.entryFormatter.format(newEntry)
+	gl.logger.Print(s)
+}
+
+type logEntry struct {
+	Time    string
+	Level   Level
+	Rlevel  Level
+	Message string
+	Data    Map
+}
+
+func newLogEntry(msg string, level Level) *logEntry {
+	return &logEntry{
+		Time:    time.Now().Format("2006-01-02T15:04:05.000-07:00"), // time.RFC3339 with milliseconds
+		Level:   level,
+		Rlevel:  NONE,
+		Message: msg,
+	}
+}
+
+func copyPairs(newEntry *logEntry, pairs []Pair) {
+	newEntry.Data = make(Map, len(pairs))
+	for _, p := range pairs {
+		newEntry.Data[p.Name] = p.Value
+	}
+}
+
+type formatter interface {
+	format(*logEntry) string
+}
+
+type textFormatter struct {
+}
+
+// ex. 2016-02-10T09:15:25.498-08:00 [INFO] This is a message from test in text format
+
+func (*textFormatter) format(newEntry *logEntry) string {
+	b := &bytes.Buffer{}
+	appendValue(b, newEntry.Time)
+	if newEntry.Rlevel != NONE {
+		fmt.Fprintf(b, "[%s,%s] ", newEntry.Level.String(), newEntry.Rlevel.String())
+	} else {
+		fmt.Fprintf(b, "[%s] ", newEntry.Level.String())
+	}
+	appendValue(b, newEntry.Message)
+	for key, value := range newEntry.Data {
+		appendKeyValue(b, key, value)
+	}
+	b.WriteByte('\n')
+	s := bytes.NewBuffer(b.Bytes())
+	return s.String()
+}
+
+func appendValue(b *bytes.Buffer, value interface{}) {
+	if _, ok := value.(string); ok {
+		fmt.Fprintf(b, "%s ", value)
+	} else {
+		fmt.Fprintf(b, "%v ", value)
+	}
+}
+
+type keyvalueFormatter struct {
+}
+
+// ex. _time=2016-02-10T09:15:25.498-08:00 _level=INFO _msg=This is a message from test in key-value format
+
+func (*keyvalueFormatter) format(newEntry *logEntry) string {
+	b := &bytes.Buffer{}
+	appendKeyValue(b, _TIME, newEntry.Time)
+	appendKeyValue(b, _LEVEL, newEntry.Level.String())
+	if newEntry.Rlevel != NONE {
+		appendKeyValue(b, _RLEVEL, newEntry.Rlevel.String())
+	}
+	appendKeyValue(b, _MSG, newEntry.Message)
+	for key, value := range newEntry.Data {
+		appendKeyValue(b, key, value)
+	}
+	b.WriteByte('\n')
+	s := bytes.NewBuffer(b.Bytes())
+	return s.String()
+}
+
+func appendKeyValue(b *bytes.Buffer, key, value interface{}) {
+	if _, ok := value.(string); ok {
+		fmt.Fprintf(b, "%v=%s ", key, value)
+	} else {
+		fmt.Fprintf(b, "%v=%v ", key, value)
+	}
+}
+
+type jsonFormatter struct {
+}
+
+// ex. {"_level":"INFO","_msg":"This is a message from test in json format","_time":"2016-02-10T09:12:59.518-08:00"}
+
+func (*jsonFormatter) format(newEntry *logEntry) string {
+	if newEntry.Data == nil {
+		newEntry.Data = make(Map, 5)
+	}
+	newEntry.Data[_TIME] = newEntry.Time
+	newEntry.Data[_LEVEL] = newEntry.Level.String()
+	if newEntry.Rlevel != NONE {
+		newEntry.Data[_RLEVEL] = newEntry.Rlevel.String()
+	}
+	newEntry.Data[_MSG] = newEntry.Message
+	serialized, _ := json.Marshal(newEntry.Data)
+	s := bytes.NewBuffer(append(serialized, '\n'))
+	return s.String()
+}

vendor/github.com/couchbase/goutils/scramsha/scramsha.go

@@ -0,0 +1,207 @@
+// @author Couchbase <info@couchbase.com>
+// @copyright 2018 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package scramsha provides implementation of client side SCRAM-SHA
+// according to https://tools.ietf.org/html/rfc5802
+package scramsha
+
+import (
+	"crypto/hmac"
+	"crypto/rand"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/base64"
+	"fmt"
+	"github.com/pkg/errors"
+	"golang.org/x/crypto/pbkdf2"
+	"hash"
+	"strconv"
+	"strings"
+)
+
+func hmacHash(message []byte, secret []byte, hashFunc func() hash.Hash) []byte {
+	h := hmac.New(hashFunc, secret)
+	h.Write(message)
+	return h.Sum(nil)
+}
+
+func shaHash(message []byte, hashFunc func() hash.Hash) []byte {
+	h := hashFunc()
+	h.Write(message)
+	return h.Sum(nil)
+}
+
+func generateClientNonce(size int) (string, error) {
+	randomBytes := make([]byte, size)
+	_, err := rand.Read(randomBytes)
+	if err != nil {
+		return "", errors.Wrap(err, "Unable to generate nonce")
+	}
+	return base64.StdEncoding.EncodeToString(randomBytes), nil
+}
+
+// ScramSha provides context for SCRAM-SHA handling
+type ScramSha struct {
+	hashSize       int
+	hashFunc       func() hash.Hash
+	clientNonce    string
+	serverNonce    string
+	salt           []byte
+	i              int
+	saltedPassword []byte
+	authMessage    string
+}
+
+var knownMethods = []string{"SCRAM-SHA512", "SCRAM-SHA256", "SCRAM-SHA1"}
+
+// BestMethod returns SCRAM-SHA method we consider the best out of suggested
+// by server
+func BestMethod(methods string) (string, error) {
+	for _, m := range knownMethods {
+		if strings.Index(methods, m) != -1 {
+			return m, nil
+		}
+	}
+	return "", errors.Errorf(
+		"None of the server suggested methods [%s] are supported",
+		methods)
+}
+
+// NewScramSha creates context for SCRAM-SHA handling
+func NewScramSha(method string) (*ScramSha, error) {
+	s := &ScramSha{}
+
+	if method == knownMethods[0] {
+		s.hashFunc = sha512.New
+		s.hashSize = 64
+	} else if method == knownMethods[1] {
+		s.hashFunc = sha256.New
+		s.hashSize = 32
+	} else if method == knownMethods[2] {
+		s.hashFunc = sha1.New
+		s.hashSize = 20
+	} else {
+		return nil, errors.Errorf("Unsupported method %s", method)
+	}
+	return s, nil
+}
+
+// GetStartRequest builds start SCRAM-SHA request to be sent to server
+func (s *ScramSha) GetStartRequest(user string) (string, error) {
+	var err error
+	s.clientNonce, err = generateClientNonce(24)
+	if err != nil {
+		return "", errors.Wrapf(err, "Unable to generate SCRAM-SHA "+
+			"start request for user %s", user)
+	}
+
+	message := fmt.Sprintf("n,,n=%s,r=%s", user, s.clientNonce)
+	s.authMessage = message[3:]
+	return message, nil
+}
+
+// HandleStartResponse handles server response on start SCRAM-SHA request
+func (s *ScramSha) HandleStartResponse(response string) error {
+	parts := strings.Split(response, ",")
+	if len(parts) != 3 {
+		return errors.Errorf("expected 3 fields in first SCRAM-SHA-1 "+
+			"server message %s", response)
+	}
+	if !strings.HasPrefix(parts[0], "r=") || len(parts[0]) < 3 {
+		return errors.Errorf("Server sent an invalid nonce %s",
+			parts[0])
+	}
+	if !strings.HasPrefix(parts[1], "s=") || len(parts[1]) < 3 {
+		return errors.Errorf("Server sent an invalid salt %s", parts[1])
+	}
+	if !strings.HasPrefix(parts[2], "i=") || len(parts[2]) < 3 {
+		return errors.Errorf("Server sent an invalid iteration count %s",
+			parts[2])
+	}
+
+	s.serverNonce = parts[0][2:]
+	encodedSalt := parts[1][2:]
+	var err error
+	s.i, err = strconv.Atoi(parts[2][2:])
+	if err != nil {
+		return errors.Errorf("Iteration count %s must be integer.",
+			parts[2][2:])
+	}
+
+	if s.i < 1 {
+		return errors.New("Iteration count should be positive")
+	}
+
+	if !strings.HasPrefix(s.serverNonce, s.clientNonce) {
+		return errors.Errorf("Server nonce %s doesn't contain client"+
+			" nonce %s", s.serverNonce, s.clientNonce)
+	}
+
+	s.salt, err = base64.StdEncoding.DecodeString(encodedSalt)
+	if err != nil {
+		return errors.Wrapf(err, "Unable to decode salt %s",
+			encodedSalt)
+	}
+
+	s.authMessage = s.authMessage + "," + response
+	return nil
+}
+
+// GetFinalRequest builds final SCRAM-SHA request to be sent to server
+func (s *ScramSha) GetFinalRequest(pass string) string {
+	clientFinalMessageBare := "c=biws,r=" + s.serverNonce
+	s.authMessage = s.authMessage + "," + clientFinalMessageBare
+
+	s.saltedPassword = pbkdf2.Key([]byte(pass), s.salt, s.i,
+		s.hashSize, s.hashFunc)
+
+	clientKey := hmacHash([]byte("Client Key"), s.saltedPassword, s.hashFunc)
+	storedKey := shaHash(clientKey, s.hashFunc)
+	clientSignature := hmacHash([]byte(s.authMessage), storedKey, s.hashFunc)
+
+	clientProof := make([]byte, len(clientSignature))
+	for i := 0; i < len(clientSignature); i++ {
+		clientProof[i] = clientKey[i] ^ clientSignature[i]
+	}
+
+	return clientFinalMessageBare + ",p=" +
+		base64.StdEncoding.EncodeToString(clientProof)
+}
+
+// HandleFinalResponse handles server's response on final SCRAM-SHA request
+func (s *ScramSha) HandleFinalResponse(response string) error {
+	if strings.Contains(response, ",") ||
+		!strings.HasPrefix(response, "v=") {
+		return errors.Errorf("Server sent an invalid final message %s",
+			response)
+	}
+
+	decodedMessage, err := base64.StdEncoding.DecodeString(response[2:])
+	if err != nil {
+		return errors.Wrapf(err, "Unable to decode server message %s",
+			response[2:])
+	}
+	serverKey := hmacHash([]byte("Server Key"), s.saltedPassword,
+		s.hashFunc)
+	serverSignature := hmacHash([]byte(s.authMessage), serverKey,
+		s.hashFunc)
+	if string(decodedMessage) != string(serverSignature) {
+		return errors.Errorf("Server proof %s doesn't match "+
+			"the expected: %s",
+			string(decodedMessage), string(serverSignature))
+	}
+	return nil
+}

vendor/github.com/couchbase/goutils/scramsha/scramsha_http.go

@@ -0,0 +1,252 @@
+// @author Couchbase <info@couchbase.com>
+// @copyright 2018 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package scramsha provides implementation of client side SCRAM-SHA
+// via Http according to https://tools.ietf.org/html/rfc7804
+package scramsha
+
+import (
+	"encoding/base64"
+	"github.com/pkg/errors"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// consts used to parse scramsha response from target
+const (
+	WWWAuthenticate    = "WWW-Authenticate"
+	AuthenticationInfo = "Authentication-Info"
+	Authorization      = "Authorization"
+	DataPrefix         = "data="
+	SidPrefix          = "sid="
+)
+
+// Request provides implementation of http request that can be retried
+type Request struct {
+	body io.ReadSeeker
+
+	// Embed an HTTP request directly. This makes a *Request act exactly
+	// like an *http.Request so that all meta methods are supported.
+	*http.Request
+}
+
+type lenReader interface {
+	Len() int
+}
+
+// NewRequest creates http request that can be retried
+func NewRequest(method, url string, body io.ReadSeeker) (*Request, error) {
+	// Wrap the body in a noop ReadCloser if non-nil. This prevents the
+	// reader from being closed by the HTTP client.
+	var rcBody io.ReadCloser
+	if body != nil {
+		rcBody = ioutil.NopCloser(body)
+	}
+
+	// Make the request with the noop-closer for the body.
+	httpReq, err := http.NewRequest(method, url, rcBody)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if we can set the Content-Length automatically.
+	if lr, ok := body.(lenReader); ok {
+		httpReq.ContentLength = int64(lr.Len())
+	}
+
+	return &Request{body, httpReq}, nil
+}
+
+func encode(str string) string {
+	return base64.StdEncoding.EncodeToString([]byte(str))
+}
+
+func decode(str string) (string, error) {
+	bytes, err := base64.StdEncoding.DecodeString(str)
+	if err != nil {
+		return "", errors.Errorf("Cannot base64 decode %s",
+			str)
+	}
+	return string(bytes), err
+}
+
+func trimPrefix(s, prefix string) (string, error) {
+	l := len(s)
+	trimmed := strings.TrimPrefix(s, prefix)
+	if l == len(trimmed) {
+		return trimmed, errors.Errorf("Prefix %s not found in %s",
+			prefix, s)
+	}
+	return trimmed, nil
+}
+
+func drainBody(resp *http.Response) {
+	defer resp.Body.Close()
+	io.Copy(ioutil.Discard, resp.Body)
+}
+
+// DoScramSha performs SCRAM-SHA handshake via Http
+func DoScramSha(req *Request,
+	username string,
+	password string,
+	client *http.Client) (*http.Response, error) {
+
+	method := "SCRAM-SHA-512"
+	s, err := NewScramSha("SCRAM-SHA512")
+	if err != nil {
+		return nil, errors.Wrap(err,
+			"Unable to initialize SCRAM-SHA handler")
+	}
+
+	message, err := s.GetStartRequest(username)
+	if err != nil {
+		return nil, err
+	}
+
+	encodedMessage := method + " " + DataPrefix + encode(message)
+
+	req.Header.Set(Authorization, encodedMessage)
+
+	res, err := client.Do(req.Request)
+	if err != nil {
+		return nil, errors.Wrap(err, "Problem sending SCRAM-SHA start"+
+			"request")
+	}
+
+	if res.StatusCode != http.StatusUnauthorized {
+		return res, nil
+	}
+
+	authHeader := res.Header.Get(WWWAuthenticate)
+	if authHeader == "" {
+		drainBody(res)
+		return nil, errors.Errorf("Header %s is not populated in "+
+			"SCRAM-SHA start response", WWWAuthenticate)
+	}
+
+	authHeader, err = trimPrefix(authHeader, method+" ")
+	if err != nil {
+		if strings.HasPrefix(authHeader, "Basic ") {
+			// user not found
+			return res, nil
+		}
+		drainBody(res)
+		return nil, errors.Wrapf(err, "Error while parsing SCRAM-SHA "+
+			"start response %s", authHeader)
+	}
+
+	drainBody(res)
+
+	sid, response, err := parseSidAndData(authHeader)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Error while parsing SCRAM-SHA "+
+			"start response %s", authHeader)
+	}
+
+	err = s.HandleStartResponse(response)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Error parsing SCRAM-SHA start "+
+			"response %s", response)
+	}
+
+	message = s.GetFinalRequest(password)
+	encodedMessage = method + " " + SidPrefix + sid + "," + DataPrefix +
+		encode(message)
+
+	req.Header.Set(Authorization, encodedMessage)
+
+	// rewind request body so it can be resent again
+	if req.body != nil {
+		if _, err = req.body.Seek(0, 0); err != nil {
+			return nil, errors.Errorf("Failed to seek body: %v",
+				err)
+		}
+	}
+
+	res, err = client.Do(req.Request)
+	if err != nil {
+		return nil, errors.Wrap(err, "Problem sending SCRAM-SHA final"+
+			"request")
+	}
+
+	if res.StatusCode == http.StatusUnauthorized {
+		// TODO retrieve and return error
+		return res, nil
+	}
+
+	if res.StatusCode >= http.StatusInternalServerError {
+		// in this case we cannot expect server to set headers properly
+		return res, nil
+	}
+
+	authHeader = res.Header.Get(AuthenticationInfo)
+	if authHeader == "" {
+		drainBody(res)
+		return nil, errors.Errorf("Header %s is not populated in "+
+			"SCRAM-SHA final response", AuthenticationInfo)
+	}
+
+	finalSid, response, err := parseSidAndData(authHeader)
+	if err != nil {
+		drainBody(res)
+		return nil, errors.Wrapf(err, "Error while parsing SCRAM-SHA "+
+			"final response %s", authHeader)
+	}
+
+	if finalSid != sid {
+		drainBody(res)
+		return nil, errors.Errorf("Sid %s returned by server "+
+			"doesn't match the original sid %s", finalSid, sid)
+	}
+
+	err = s.HandleFinalResponse(response)
+	if err != nil {
+		drainBody(res)
+		return nil, errors.Wrapf(err,
+			"Error handling SCRAM-SHA final server response %s",
+			response)
+	}
+	return res, nil
+}
+
+func parseSidAndData(authHeader string) (string, string, error) {
+	sidIndex := strings.Index(authHeader, SidPrefix)
+	if sidIndex < 0 {
+		return "", "", errors.Errorf("Cannot find %s in %s",
+			SidPrefix, authHeader)
+	}
+
+	sidEndIndex := strings.Index(authHeader, ",")
+	if sidEndIndex < 0 {
+		return "", "", errors.Errorf("Cannot find ',' in %s",
+			authHeader)
+	}
+
+	sid := authHeader[sidIndex+len(SidPrefix) : sidEndIndex]
+
+	dataIndex := strings.Index(authHeader, DataPrefix)
+	if dataIndex < 0 {
+		return "", "", errors.Errorf("Cannot find %s in %s",
+			DataPrefix, authHeader)
+	}
+
+	data, err := decode(authHeader[dataIndex+len(DataPrefix):])
+	if err != nil {
+		return "", "", err
+	}
+	return sid, data, nil
+}

vendor/github.com/couchbaselabs/go-couchbase/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2013  Couchbase, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/couchbaselabs/go-couchbase/audit.go

@@ -0,0 +1,32 @@
+package couchbase
+
+import ()
+
+// Sample data:
+// {"disabled":["12333", "22244"],"uid":"132492431","auditdEnabled":true,
+//  "disabledUsers":[{"name":"bill","domain":"local"},{"name":"bob","domain":"local"}],
+//  "logPath":"/Users/johanlarson/Library/Application Support/Couchbase/var/lib/couchbase/logs",
+//  "rotateInterval":86400,"rotateSize":20971520}
+type AuditSpec struct {
+	Disabled       []uint32    `json:"disabled"`
+	Uid            string      `json:"uid"`
+	AuditdEnabled  bool        `json:"auditdEnabled`
+	DisabledUsers  []AuditUser `json:"disabledUsers"`
+	LogPath        string      `json:"logPath"`
+	RotateInterval int64       `json:"rotateInterval"`
+	RotateSize     int64       `json:"rotateSize"`
+}
+
+type AuditUser struct {
+	Name   string `json:"name"`
+	Domain string `json:"domain"`
+}
+
+func (c *Client) GetAuditSpec() (*AuditSpec, error) {
+	ret := &AuditSpec{}
+	err := c.parseURLResponse("/settings/audit", ret)
+	if err != nil {
+		return nil, err
+	}
+	return ret, nil
+}

vendor/github.com/couchbaselabs/go-couchbase/conn_pool.go

@@ -0,0 +1,387 @@
+package couchbase
+
+import (
+	"errors"
+	"sync/atomic"
+	"time"
+
+	"github.com/couchbase/gomemcached"
+	"github.com/couchbase/gomemcached/client"
+	"github.com/couchbase/goutils/logging"
+)
+
+// GenericMcdAuthHandler is a kind of AuthHandler that performs
+// special auth exchange (like non-standard auth, possibly followed by
+// select-bucket).
+type GenericMcdAuthHandler interface {
+	AuthHandler
+	AuthenticateMemcachedConn(host string, conn *memcached.Client) error
+}
+
+// Error raised when a connection can't be retrieved from a pool.
+var TimeoutError = errors.New("timeout waiting to build connection")
+var errClosedPool = errors.New("the connection pool is closed")
+var errNoPool = errors.New("no connection pool")
+
+// Default timeout for retrieving a connection from the pool.
+var ConnPoolTimeout = time.Hour * 24 * 30
+
+// overflow connection closer cycle time
+var ConnCloserInterval = time.Second * 30
+
+// ConnPoolAvailWaitTime is the amount of time to wait for an existing
+// connection from the pool before considering the creation of a new
+// one.
+var ConnPoolAvailWaitTime = time.Millisecond
+
+type connectionPool struct {
+	host        string
+	mkConn      func(host string, ah AuthHandler) (*memcached.Client, error)
+	auth        AuthHandler
+	connections chan *memcached.Client
+	createsem   chan bool
+	bailOut     chan bool
+	poolSize    int
+	connCount   uint64
+	inUse       bool
+}
+
+func newConnectionPool(host string, ah AuthHandler, closer bool, poolSize, poolOverflow int) *connectionPool {
+	connSize := poolSize
+	if closer {
+		connSize += poolOverflow
+	}
+	rv := &connectionPool{
+		host:        host,
+		connections: make(chan *memcached.Client, connSize),
+		createsem:   make(chan bool, poolSize+poolOverflow),
+		mkConn:      defaultMkConn,
+		auth:        ah,
+		poolSize:    poolSize,
+	}
+	if closer {
+		rv.bailOut = make(chan bool, 1)
+		go rv.connCloser()
+	}
+	return rv
+}
+
+// ConnPoolTimeout is notified whenever connections are acquired from a pool.
+var ConnPoolCallback func(host string, source string, start time.Time, err error)
+
+func defaultMkConn(host string, ah AuthHandler) (*memcached.Client, error) {
+	var features memcached.Features
+
+	conn, err := memcached.Connect("tcp", host)
+	if err != nil {
+		return nil, err
+	}
+
+	if TCPKeepalive == true {
+		conn.SetKeepAliveOptions(time.Duration(TCPKeepaliveInterval) * time.Second)
+	}
+
+	if EnableMutationToken == true {
+		features = append(features, memcached.FeatureMutationToken)
+	}
+	if EnableDataType == true {
+		features = append(features, memcached.FeatureDataType)
+	}
+
+	if EnableXattr == true {
+		features = append(features, memcached.FeatureXattr)
+	}
+
+	if len(features) > 0 {
+		if DefaultTimeout > 0 {
+			conn.SetDeadline(getDeadline(noDeadline, DefaultTimeout))
+		}
+
+		res, err := conn.EnableFeatures(features)
+
+		if DefaultTimeout > 0 {
+			conn.SetDeadline(noDeadline)
+		}
+
+		if err != nil && isTimeoutError(err) {
+			conn.Close()
+			return nil, err
+		}
+
+		if err != nil || res.Status != gomemcached.SUCCESS {
+			logging.Warnf("Unable to enable features %v", err)
+		}
+	}
+
+	if gah, ok := ah.(GenericMcdAuthHandler); ok {
+		err = gah.AuthenticateMemcachedConn(host, conn)
+		if err != nil {
+			conn.Close()
+			return nil, err
+		}
+		return conn, nil
+	}
+	name, pass, bucket := ah.GetCredentials()
+	if name != "default" {
+		_, err = conn.Auth(name, pass)
+		if err != nil {
+			conn.Close()
+			return nil, err
+		}
+		// Select bucket (Required for cb_auth creds)
+		// Required when doing auth with _admin credentials
+		if bucket != "" && bucket != name {
+			_, err = conn.SelectBucket(bucket)
+			if err != nil {
+				conn.Close()
+				return nil, err
+			}
+		}
+	}
+	return conn, nil
+}
+
+func (cp *connectionPool) Close() (err error) {
+	defer func() {
+		if recover() != nil {
+			err = errors.New("connectionPool.Close error")
+		}
+	}()
+	if cp.bailOut != nil {
+
+		// defensively, we won't wait if the channel is full
+		select {
+		case cp.bailOut <- false:
+		default:
+		}
+	}
+	close(cp.connections)
+	for c := range cp.connections {
+		c.Close()
+	}
+	return
+}
+
+func (cp *connectionPool) Node() string {
+	return cp.host
+}
+
+func (cp *connectionPool) GetWithTimeout(d time.Duration) (rv *memcached.Client, err error) {
+	if cp == nil {
+		return nil, errNoPool
+	}
+
+	path := ""
+
+	if ConnPoolCallback != nil {
+		defer func(path *string, start time.Time) {
+			ConnPoolCallback(cp.host, *path, start, err)
+		}(&path, time.Now())
+	}
+
+	path = "short-circuit"
+
+	// short-circuit available connetions.
+	select {
+	case rv, isopen := <-cp.connections:
+		if !isopen {
+			return nil, errClosedPool
+		}
+		atomic.AddUint64(&cp.connCount, 1)
+		return rv, nil
+	default:
+	}
+
+	t := time.NewTimer(ConnPoolAvailWaitTime)
+	defer t.Stop()
+
+	// Try to grab an available connection within 1ms
+	select {
+	case rv, isopen := <-cp.connections:
+		path = "avail1"
+		if !isopen {
+			return nil, errClosedPool
+		}
+		atomic.AddUint64(&cp.connCount, 1)
+		return rv, nil
+	case <-t.C:
+		// No connection came around in time, let's see
+		// whether we can get one or build a new one first.
+		t.Reset(d) // Reuse the timer for the full timeout.
+		select {
+		case rv, isopen := <-cp.connections:
+			path = "avail2"
+			if !isopen {
+				return nil, errClosedPool
+			}
+			atomic.AddUint64(&cp.connCount, 1)
+			return rv, nil
+		case cp.createsem <- true:
+			path = "create"
+			// Build a connection if we can't get a real one.
+			// This can potentially be an overflow connection, or
+			// a pooled connection.
+			rv, err := cp.mkConn(cp.host, cp.auth)
+			if err != nil {
+				// On error, release our create hold
+				<-cp.createsem
+			} else {
+				atomic.AddUint64(&cp.connCount, 1)
+			}
+			return rv, err
+		case <-t.C:
+			return nil, ErrTimeout
+		}
+	}
+}
+
+func (cp *connectionPool) Get() (*memcached.Client, error) {
+	return cp.GetWithTimeout(ConnPoolTimeout)
+}
+
+func (cp *connectionPool) Return(c *memcached.Client) {
+	if c == nil {
+		return
+	}
+
+	if cp == nil {
+		c.Close()
+	}
+
+	if c.IsHealthy() {
+		defer func() {
+			if recover() != nil {
+				// This happens when the pool has already been
+				// closed and we're trying to return a
+				// connection to it anyway.  Just close the
+				// connection.
+				c.Close()
+			}
+		}()
+
+		select {
+		case cp.connections <- c:
+		default:
+			<-cp.createsem
+			c.Close()
+		}
+	} else {
+		<-cp.createsem
+		c.Close()
+	}
+}
+
+// give the ability to discard a connection from a pool
+// useful for ditching connections to the wrong node after a rebalance
+func (cp *connectionPool) Discard(c *memcached.Client) {
+	<-cp.createsem
+	c.Close()
+}
+
+// asynchronous connection closer
+func (cp *connectionPool) connCloser() {
+	var connCount uint64
+
+	t := time.NewTimer(ConnCloserInterval)
+	defer t.Stop()
+
+	for {
+		connCount = cp.connCount
+
+		// we don't exist anymore! bail out!
+		select {
+		case <-cp.bailOut:
+			return
+		case <-t.C:
+		}
+		t.Reset(ConnCloserInterval)
+
+		// no overflow connections open or sustained requests for connections
+		// nothing to do until the next cycle
+		if len(cp.connections) <= cp.poolSize ||
+			ConnCloserInterval/ConnPoolAvailWaitTime < time.Duration(cp.connCount-connCount) {
+			continue
+		}
+
+		// close overflow connections now that they are not needed
+		for c := range cp.connections {
+			select {
+			case <-cp.bailOut:
+				return
+			default:
+			}
+
+			// bail out if close did not work out
+			if !cp.connCleanup(c) {
+				return
+			}
+			if len(cp.connections) <= cp.poolSize {
+				break
+			}
+		}
+	}
+}
+
+// close connection with recovery on error
+func (cp *connectionPool) connCleanup(c *memcached.Client) (rv bool) {
+
+	// just in case we are closing a connection after
+	// bailOut has been sent but we haven't yet read it
+	defer func() {
+		if recover() != nil {
+			rv = false
+		}
+	}()
+	rv = true
+
+	c.Close()
+	<-cp.createsem
+	return
+}
+
+func (cp *connectionPool) StartTapFeed(args *memcached.TapArguments) (*memcached.TapFeed, error) {
+	if cp == nil {
+		return nil, errNoPool
+	}
+	mc, err := cp.Get()
+	if err != nil {
+		return nil, err
+	}
+
+	// A connection can't be used after TAP; Dont' count it against the
+	// connection pool capacity
+	<-cp.createsem
+
+	return mc.StartTapFeed(*args)
+}
+
+const DEFAULT_WINDOW_SIZE = 20 * 1024 * 1024 // 20 Mb
+
+func (cp *connectionPool) StartUprFeed(name string, sequence uint32, dcp_buffer_size uint32, data_chan_size int) (*memcached.UprFeed, error) {
+	if cp == nil {
+		return nil, errNoPool
+	}
+	mc, err := cp.Get()
+	if err != nil {
+		return nil, err
+	}
+
+	// A connection can't be used after it has been allocated to UPR;
+	// Dont' count it against the connection pool capacity
+	<-cp.createsem
+
+	uf, err := mc.NewUprFeed()
+	if err != nil {
+		return nil, err
+	}
+
+	if err := uf.UprOpen(name, sequence, dcp_buffer_size); err != nil {
+		return nil, err
+	}
+
+	if err := uf.StartFeedWithConfig(data_chan_size); err != nil {
+		return nil, err
+	}
+
+	return uf, nil
+}

vendor/github.com/couchbaselabs/go-couchbase/ddocs.go

@@ -0,0 +1,288 @@
+package couchbase
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/couchbase/goutils/logging"
+	"io/ioutil"
+	"net/http"
+)
+
+// ViewDefinition represents a single view within a design document.
+type ViewDefinition struct {
+	Map    string `json:"map"`
+	Reduce string `json:"reduce,omitempty"`
+}
+
+// DDoc is the document body of a design document specifying a view.
+type DDoc struct {
+	Language string                    `json:"language,omitempty"`
+	Views    map[string]ViewDefinition `json:"views"`
+}
+
+// DDocsResult represents the result from listing the design
+// documents.
+type DDocsResult struct {
+	Rows []struct {
+		DDoc struct {
+			Meta map[string]interface{}
+			JSON DDoc
+		} `json:"doc"`
+	} `json:"rows"`
+}
+
+// GetDDocs lists all design documents
+func (b *Bucket) GetDDocs() (DDocsResult, error) {
+	var ddocsResult DDocsResult
+	b.RLock()
+	pool := b.pool
+	uri := b.DDocs.URI
+	b.RUnlock()
+
+	// MB-23555 ephemeral buckets have no ddocs
+	if uri == "" {
+		return DDocsResult{}, nil
+	}
+
+	err := pool.client.parseURLResponse(uri, &ddocsResult)
+	if err != nil {
+		return DDocsResult{}, err
+	}
+	return ddocsResult, nil
+}
+
+func (b *Bucket) GetDDocWithRetry(docname string, into interface{}) error {
+	ddocURI := fmt.Sprintf("/%s/_design/%s", b.GetName(), docname)
+	err := b.parseAPIResponse(ddocURI, &into)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (b *Bucket) GetDDocsWithRetry() (DDocsResult, error) {
+	var ddocsResult DDocsResult
+	b.RLock()
+	uri := b.DDocs.URI
+	b.RUnlock()
+
+	// MB-23555 ephemeral buckets have no ddocs
+	if uri == "" {
+		return DDocsResult{}, nil
+	}
+
+	err := b.parseURLResponse(uri, &ddocsResult)
+	if err != nil {
+		return DDocsResult{}, err
+	}
+	return ddocsResult, nil
+}
+
+func (b *Bucket) ddocURL(docname string) (string, error) {
+	u, err := b.randomBaseURL()
+	if err != nil {
+		return "", err
+	}
+	u.Path = fmt.Sprintf("/%s/_design/%s", b.GetName(), docname)
+	return u.String(), nil
+}
+
+func (b *Bucket) ddocURLNext(nodeId int, docname string) (string, int, error) {
+	u, selected, err := b.randomNextURL(nodeId)
+	if err != nil {
+		return "", -1, err
+	}
+	u.Path = fmt.Sprintf("/%s/_design/%s", b.GetName(), docname)
+	return u.String(), selected, nil
+}
+
+const ABS_MAX_RETRIES = 10
+const ABS_MIN_RETRIES = 3
+
+func (b *Bucket) getMaxRetries() (int, error) {
+
+	maxRetries := len(b.Nodes())
+
+	if maxRetries == 0 {
+		return 0, fmt.Errorf("No available Couch rest URLs")
+	}
+
+	if maxRetries > ABS_MAX_RETRIES {
+		maxRetries = ABS_MAX_RETRIES
+	} else if maxRetries < ABS_MIN_RETRIES {
+		maxRetries = ABS_MIN_RETRIES
+	}
+
+	return maxRetries, nil
+}
+
+// PutDDoc installs a design document.
+func (b *Bucket) PutDDoc(docname string, value interface{}) error {
+
+	var Err error
+
+	maxRetries, err := b.getMaxRetries()
+	if err != nil {
+		return err
+	}
+
+	lastNode := START_NODE_ID
+
+	for retryCount := 0; retryCount < maxRetries; retryCount++ {
+
+		Err = nil
+
+		ddocU, selectedNode, err := b.ddocURLNext(lastNode, docname)
+		if err != nil {
+			return err
+		}
+
+		lastNode = selectedNode
+
+		logging.Infof(" Trying with selected node %d", selectedNode)
+		j, err := json.Marshal(value)
+		if err != nil {
+			return err
+		}
+
+		req, err := http.NewRequest("PUT", ddocU, bytes.NewReader(j))
+		if err != nil {
+			return err
+		}
+		req.Header.Set("Content-Type", "application/json")
+		err = maybeAddAuth(req, b.authHandler(false /* bucket not yet locked */))
+		if err != nil {
+			return err
+		}
+
+		res, err := doHTTPRequest(req)
+		if err != nil {
+			return err
+		}
+
+		if res.StatusCode != 201 {
+			body, _ := ioutil.ReadAll(res.Body)
+			Err = fmt.Errorf("error installing view: %v / %s",
+				res.Status, body)
+			logging.Errorf(" Error in PutDDOC %v. Retrying...", Err)
+			res.Body.Close()
+			b.Refresh()
+			continue
+		}
+
+		res.Body.Close()
+		break
+	}
+
+	return Err
+}
+
+// GetDDoc retrieves a specific a design doc.
+func (b *Bucket) GetDDoc(docname string, into interface{}) error {
+	var Err error
+	var res *http.Response
+
+	maxRetries, err := b.getMaxRetries()
+	if err != nil {
+		return err
+	}
+
+	lastNode := START_NODE_ID
+	for retryCount := 0; retryCount < maxRetries; retryCount++ {
+
+		Err = nil
+		ddocU, selectedNode, err := b.ddocURLNext(lastNode, docname)
+		if err != nil {
+			return err
+		}
+
+		lastNode = selectedNode
+		logging.Infof(" Trying with selected node %d", selectedNode)
+
+		req, err := http.NewRequest("GET", ddocU, nil)
+		if err != nil {
+			return err
+		}
+		req.Header.Set("Content-Type", "application/json")
+		err = maybeAddAuth(req, b.authHandler(false /* bucket not yet locked */))
+		if err != nil {
+			return err
+		}
+
+		res, err = doHTTPRequest(req)
+		if err != nil {
+			return err
+		}
+		if res.StatusCode != 200 {
+			body, _ := ioutil.ReadAll(res.Body)
+			Err = fmt.Errorf("error reading view: %v / %s",
+				res.Status, body)
+			logging.Errorf(" Error in GetDDOC %v Retrying...", Err)
+			b.Refresh()
+			res.Body.Close()
+			continue
+		}
+		defer res.Body.Close()
+		break
+	}
+
+	if Err != nil {
+		return Err
+	}
+
+	d := json.NewDecoder(res.Body)
+	return d.Decode(into)
+}
+
+// DeleteDDoc removes a design document.
+func (b *Bucket) DeleteDDoc(docname string) error {
+
+	var Err error
+
+	maxRetries, err := b.getMaxRetries()
+	if err != nil {
+		return err
+	}
+
+	lastNode := START_NODE_ID
+
+	for retryCount := 0; retryCount < maxRetries; retryCount++ {
+
+		Err = nil
+		ddocU, selectedNode, err := b.ddocURLNext(lastNode, docname)
+		if err != nil {
+			return err
+		}
+
+		lastNode = selectedNode
+		logging.Infof(" Trying with selected node %d", selectedNode)
+
+		req, err := http.NewRequest("DELETE", ddocU, nil)
+		if err != nil {
+			return err
+		}
+		req.Header.Set("Content-Type", "application/json")
+		err = maybeAddAuth(req, b.authHandler(false /* bucket not already locked */))
+		if err != nil {
+			return err
+		}
+
+		res, err := doHTTPRequest(req)
+		if err != nil {
+			return err
+		}
+		if res.StatusCode != 200 {
+			body, _ := ioutil.ReadAll(res.Body)
+			Err = fmt.Errorf("error deleting view : %v / %s", res.Status, body)
+			logging.Errorf(" Error in DeleteDDOC %v. Retrying ... ", Err)
+			b.Refresh()
+			res.Body.Close()
+			continue
+		}
+
+		res.Body.Close()
+		break
+	}
+	return Err
+}

vendor/github.com/couchbaselabs/go-couchbase/observe.go

@@ -0,0 +1,300 @@
+package couchbase
+
+import (
+	"fmt"
+	"github.com/couchbase/goutils/logging"
+	"sync"
+)
+
+type PersistTo uint8
+
+const (
+	PersistNone   = PersistTo(0x00)
+	PersistMaster = PersistTo(0x01)
+	PersistOne    = PersistTo(0x02)
+	PersistTwo    = PersistTo(0x03)
+	PersistThree  = PersistTo(0x04)
+	PersistFour   = PersistTo(0x05)
+)
+
+type ObserveTo uint8
+
+const (
+	ObserveNone           = ObserveTo(0x00)
+	ObserveReplicateOne   = ObserveTo(0x01)
+	ObserveReplicateTwo   = ObserveTo(0x02)
+	ObserveReplicateThree = ObserveTo(0x03)
+	ObserveReplicateFour  = ObserveTo(0x04)
+)
+
+type JobType uint8
+
+const (
+	OBSERVE = JobType(0x00)
+	PERSIST = JobType(0x01)
+)
+
+type ObservePersistJob struct {
+	vb                 uint16
+	vbuuid             uint64
+	hostname           string
+	jobType            JobType
+	failover           uint8
+	lastPersistedSeqNo uint64
+	currentSeqNo       uint64
+	resultChan         chan *ObservePersistJob
+	errorChan          chan *OPErrResponse
+}
+
+type OPErrResponse struct {
+	vb     uint16
+	vbuuid uint64
+	err    error
+	job    *ObservePersistJob
+}
+
+var ObservePersistPool = NewPool(1024)
+var OPJobChan = make(chan *ObservePersistJob, 1024)
+var OPJobDone = make(chan bool)
+
+var wg sync.WaitGroup
+
+func (b *Bucket) StartOPPollers(maxWorkers int) {
+
+	for i := 0; i < maxWorkers; i++ {
+		go b.OPJobPoll()
+		wg.Add(1)
+	}
+	wg.Wait()
+}
+
+func (b *Bucket) SetObserveAndPersist(nPersist PersistTo, nObserve ObserveTo) (err error) {
+
+	numNodes := len(b.Nodes())
+	if int(nPersist) > numNodes || int(nObserve) > numNodes {
+		return fmt.Errorf("Not enough healthy nodes in the cluster")
+	}
+
+	if int(nPersist) > (b.Replicas+1) || int(nObserve) > b.Replicas {
+		return fmt.Errorf("Not enough replicas in the cluster")
+	}
+
+	if EnableMutationToken == false {
+		return fmt.Errorf("Mutation Tokens not enabled ")
+	}
+
+	b.ds = &DurablitySettings{Persist: PersistTo(nPersist), Observe: ObserveTo(nObserve)}
+	return
+}
+
+func (b *Bucket) ObserveAndPersistPoll(vb uint16, vbuuid uint64, seqNo uint64) (err error, failover bool) {
+	b.RLock()
+	ds := b.ds
+	b.RUnlock()
+
+	if ds == nil {
+		return
+	}
+
+	nj := 0 // total number of jobs
+	resultChan := make(chan *ObservePersistJob, 10)
+	errChan := make(chan *OPErrResponse, 10)
+
+	nodes := b.GetNodeList(vb)
+	if int(ds.Observe) > len(nodes) || int(ds.Persist) > len(nodes) {
+		return fmt.Errorf("Not enough healthy nodes in the cluster"), false
+	}
+
+	logging.Infof("Node list %v", nodes)
+
+	if ds.Observe >= ObserveReplicateOne {
+		// create a job for each host
+		for i := ObserveReplicateOne; i < ds.Observe+1; i++ {
+			opJob := ObservePersistPool.Get()
+			opJob.vb = vb
+			opJob.vbuuid = vbuuid
+			opJob.jobType = OBSERVE
+			opJob.hostname = nodes[i]
+			opJob.resultChan = resultChan
+			opJob.errorChan = errChan
+
+			OPJobChan <- opJob
+			nj++
+
+		}
+	}
+
+	if ds.Persist >= PersistMaster {
+		for i := PersistMaster; i < ds.Persist+1; i++ {
+			opJob := ObservePersistPool.Get()
+			opJob.vb = vb
+			opJob.vbuuid = vbuuid
+			opJob.jobType = PERSIST
+			opJob.hostname = nodes[i]
+			opJob.resultChan = resultChan
+			opJob.errorChan = errChan
+
+			OPJobChan <- opJob
+			nj++
+
+		}
+	}
+
+	ok := true
+	for ok {
+		select {
+		case res := <-resultChan:
+			jobDone := false
+			if res.failover == 0 {
+				// no failover
+				if res.jobType == PERSIST {
+					if res.lastPersistedSeqNo >= seqNo {
+						jobDone = true
+					}
+
+				} else {
+					if res.currentSeqNo >= seqNo {
+						jobDone = true
+					}
+				}
+
+				if jobDone == true {
+					nj--
+					ObservePersistPool.Put(res)
+				} else {
+					// requeue this job
+					OPJobChan <- res
+				}
+
+			} else {
+				// Not currently handling failover scenarios TODO
+				nj--
+				ObservePersistPool.Put(res)
+				failover = true
+			}
+
+			if nj == 0 {
+				// done with all the jobs
+				ok = false
+				close(resultChan)
+				close(errChan)
+			}
+
+		case Err := <-errChan:
+			logging.Errorf("Error in Observe/Persist %v", Err.err)
+			err = fmt.Errorf("Error in Observe/Persist job %v", Err.err)
+			nj--
+			ObservePersistPool.Put(Err.job)
+			if nj == 0 {
+				close(resultChan)
+				close(errChan)
+				ok = false
+			}
+		}
+	}
+
+	return
+}
+
+func (b *Bucket) OPJobPoll() {
+
+	ok := true
+	for ok == true {
+		select {
+		case job := <-OPJobChan:
+			pool := b.getConnPoolByHost(job.hostname, false /* bucket not already locked */)
+			if pool == nil {
+				errRes := &OPErrResponse{vb: job.vb, vbuuid: job.vbuuid}
+				errRes.err = fmt.Errorf("Pool not found for host %v", job.hostname)
+				errRes.job = job
+				job.errorChan <- errRes
+				continue
+			}
+			conn, err := pool.Get()
+			if err != nil {
+				errRes := &OPErrResponse{vb: job.vb, vbuuid: job.vbuuid}
+				errRes.err = fmt.Errorf("Unable to get connection from pool %v", err)
+				errRes.job = job
+				job.errorChan <- errRes
+				continue
+			}
+
+			res, err := conn.ObserveSeq(job.vb, job.vbuuid)
+			if err != nil {
+				errRes := &OPErrResponse{vb: job.vb, vbuuid: job.vbuuid}
+				errRes.err = fmt.Errorf("Command failed %v", err)
+				errRes.job = job
+				job.errorChan <- errRes
+				continue
+
+			}
+			pool.Return(conn)
+			job.lastPersistedSeqNo = res.LastPersistedSeqNo
+			job.currentSeqNo = res.CurrentSeqNo
+			job.failover = res.Failover
+
+			job.resultChan <- job
+		case <-OPJobDone:
+			logging.Infof("Observe Persist Poller exitting")
+			ok = false
+		}
+	}
+	wg.Done()
+}
+
+func (b *Bucket) GetNodeList(vb uint16) []string {
+
+	vbm := b.VBServerMap()
+	if len(vbm.VBucketMap) < int(vb) {
+		logging.Infof("vbmap smaller than vblist")
+		return nil
+	}
+
+	nodes := make([]string, len(vbm.VBucketMap[vb]))
+	for i := 0; i < len(vbm.VBucketMap[vb]); i++ {
+		n := vbm.VBucketMap[vb][i]
+		if n < 0 {
+			continue
+		}
+
+		node := b.getMasterNode(n)
+		if len(node) > 1 {
+			nodes[i] = node
+		}
+		continue
+
+	}
+	return nodes
+}
+
+//pool of ObservePersist Jobs
+type OPpool struct {
+	pool chan *ObservePersistJob
+}
+
+// NewPool creates a new pool of jobs
+func NewPool(max int) *OPpool {
+	return &OPpool{
+		pool: make(chan *ObservePersistJob, max),
+	}
+}
+
+// Borrow a Client from the pool.
+func (p *OPpool) Get() *ObservePersistJob {
+	var o *ObservePersistJob
+	select {
+	case o = <-p.pool:
+	default:
+		o = &ObservePersistJob{}
+	}
+	return o
+}
+
+// Return returns a Client to the pool.
+func (p *OPpool) Put(o *ObservePersistJob) {
+	select {
+	case p.pool <- o:
+	default:
+		// let it go, let it go...
+	}
+}

vendor/github.com/couchbaselabs/go-couchbase/streaming.go

@@ -0,0 +1,209 @@
+package couchbase
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/couchbase/goutils/logging"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net"
+	"net/http"
+	"time"
+	"unsafe"
+)
+
+// Bucket auto-updater gets the latest version of the bucket config from
+// the server. If the configuration has changed then updated the local
+// bucket information. If the bucket has been deleted then notify anyone
+// who is holding a reference to this bucket
+
+const MAX_RETRY_COUNT = 5
+const DISCONNECT_PERIOD = 120 * time.Second
+
+type NotifyFn func(bucket string, err error)
+
+// Use TCP keepalive to detect half close sockets
+var updaterTransport http.RoundTripper = &http.Transport{
+	Proxy: http.ProxyFromEnvironment,
+	Dial: (&net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+	}).Dial,
+}
+
+var updaterHTTPClient = &http.Client{Transport: updaterTransport}
+
+func doHTTPRequestForUpdate(req *http.Request) (*http.Response, error) {
+
+	var err error
+	var res *http.Response
+
+	for i := 0; i < HTTP_MAX_RETRY; i++ {
+		res, err = updaterHTTPClient.Do(req)
+		if err != nil && isHttpConnError(err) {
+			continue
+		}
+		break
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return res, err
+}
+
+func (b *Bucket) RunBucketUpdater(notify NotifyFn) {
+	go func() {
+		err := b.UpdateBucket()
+		if err != nil {
+			if notify != nil {
+				notify(b.GetName(), err)
+			}
+			logging.Errorf(" Bucket Updater exited with err %v", err)
+		}
+	}()
+}
+
+func (b *Bucket) replaceConnPools2(with []*connectionPool, bucketLocked bool) {
+	if !bucketLocked {
+		b.Lock()
+		defer b.Unlock()
+	}
+	old := b.connPools
+	b.connPools = unsafe.Pointer(&with)
+	if old != nil {
+		for _, pool := range *(*[]*connectionPool)(old) {
+			if pool != nil && pool.inUse == false {
+				pool.Close()
+			}
+		}
+	}
+	return
+}
+
+func (b *Bucket) UpdateBucket() error {
+
+	var failures int
+	var returnErr error
+
+	for {
+
+		if failures == MAX_RETRY_COUNT {
+			logging.Errorf(" Maximum failures reached. Exiting loop...")
+			return fmt.Errorf("Max failures reached. Last Error %v", returnErr)
+		}
+
+		nodes := b.Nodes()
+		if len(nodes) < 1 {
+			return fmt.Errorf("No healthy nodes found")
+		}
+
+		startNode := rand.Intn(len(nodes))
+		node := nodes[(startNode)%len(nodes)]
+
+		streamUrl := fmt.Sprintf("http://%s/pools/default/bucketsStreaming/%s", node.Hostname, b.GetName())
+		logging.Infof(" Trying with %s", streamUrl)
+		req, err := http.NewRequest("GET", streamUrl, nil)
+		if err != nil {
+			return err
+		}
+
+		b.RLock()
+		pool := b.pool
+		bucketName := b.Name
+		b.RUnlock()
+		scopes, err := getScopesAndCollections(pool, bucketName)
+		if err != nil {
+			return err
+		}
+
+		// Lock here to avoid having pool closed under us.
+		b.RLock()
+		err = maybeAddAuth(req, b.pool.client.ah)
+		b.RUnlock()
+		if err != nil {
+			return err
+		}
+
+		res, err := doHTTPRequestForUpdate(req)
+		if err != nil {
+			return err
+		}
+
+		if res.StatusCode != 200 {
+			bod, _ := ioutil.ReadAll(io.LimitReader(res.Body, 512))
+			logging.Errorf("Failed to connect to host, unexpected status code: %v. Body %s", res.StatusCode, bod)
+			res.Body.Close()
+			returnErr = fmt.Errorf("Failed to connect to host. Status %v Body %s", res.StatusCode, bod)
+			failures++
+			continue
+		}
+
+		dec := json.NewDecoder(res.Body)
+
+		tmpb := &Bucket{}
+		for {
+
+			err := dec.Decode(&tmpb)
+			if err != nil {
+				returnErr = err
+				res.Body.Close()
+				break
+			}
+
+			// if we got here, reset failure count
+			failures = 0
+			b.Lock()
+
+			// mark all the old connection pools for deletion
+			pools := b.getConnPools(true /* already locked */)
+			for _, pool := range pools {
+				if pool != nil {
+					pool.inUse = false
+				}
+			}
+
+			newcps := make([]*connectionPool, len(tmpb.VBSMJson.ServerList))
+			for i := range newcps {
+				// get the old connection pool and check if it is still valid
+				pool := b.getConnPoolByHost(tmpb.VBSMJson.ServerList[i], true /* bucket already locked */)
+				if pool != nil && pool.inUse == false {
+					// if the hostname and index is unchanged then reuse this pool
+					newcps[i] = pool
+					pool.inUse = true
+					continue
+				}
+				// else create a new pool
+				if b.ah != nil {
+					newcps[i] = newConnectionPool(
+						tmpb.VBSMJson.ServerList[i],
+						b.ah, false, PoolSize, PoolOverflow)
+
+				} else {
+					newcps[i] = newConnectionPool(
+						tmpb.VBSMJson.ServerList[i],
+						b.authHandler(true /* bucket already locked */),
+						false, PoolSize, PoolOverflow)
+				}
+			}
+
+			b.replaceConnPools2(newcps, true /* bucket already locked */)
+
+			tmpb.ah = b.ah
+			b.vBucketServerMap = unsafe.Pointer(&tmpb.VBSMJson)
+			b.nodeList = unsafe.Pointer(&tmpb.NodesJSON)
+			b.Scopes = scopes
+			b.Unlock()
+
+			logging.Infof("Got new configuration for bucket %s", b.GetName())
+
+		}
+		// we are here because of an error
+		failures++
+		continue
+
+	}
+	return nil
+}

vendor/github.com/couchbaselabs/go-couchbase/tap.go

@@ -0,0 +1,143 @@
+package couchbase
+
+import (
+	"github.com/couchbase/gomemcached/client"
+	"github.com/couchbase/goutils/logging"
+	"sync"
+	"time"
+)
+
+const initialRetryInterval = 1 * time.Second
+const maximumRetryInterval = 30 * time.Second
+
+// A TapFeed streams mutation events from a bucket.
+//
+// Events from the bucket can be read from the channel 'C'.  Remember
+// to call Close() on it when you're done, unless its channel has
+// closed itself already.
+type TapFeed struct {
+	C <-chan memcached.TapEvent
+
+	bucket    *Bucket
+	args      *memcached.TapArguments
+	nodeFeeds []*memcached.TapFeed    // The TAP feeds of the individual nodes
+	output    chan memcached.TapEvent // Same as C but writeably-typed
+	wg        sync.WaitGroup
+	quit      chan bool
+}
+
+// StartTapFeed creates and starts a new Tap feed
+func (b *Bucket) StartTapFeed(args *memcached.TapArguments) (*TapFeed, error) {
+	if args == nil {
+		defaultArgs := memcached.DefaultTapArguments()
+		args = &defaultArgs
+	}
+
+	feed := &TapFeed{
+		bucket: b,
+		args:   args,
+		output: make(chan memcached.TapEvent, 10),
+		quit:   make(chan bool),
+	}
+
+	go feed.run()
+
+	feed.C = feed.output
+	return feed, nil
+}
+
+// Goroutine that runs the feed
+func (feed *TapFeed) run() {
+	retryInterval := initialRetryInterval
+	bucketOK := true
+	for {
+		// Connect to the TAP feed of each server node:
+		if bucketOK {
+			killSwitch, err := feed.connectToNodes()
+			if err == nil {
+				// Run until one of the sub-feeds fails:
+				select {
+				case <-killSwitch:
+				case <-feed.quit:
+					return
+				}
+				feed.closeNodeFeeds()
+				retryInterval = initialRetryInterval
+			}
+		}
+
+		// On error, try to refresh the bucket in case the list of nodes changed:
+		logging.Infof("go-couchbase: TAP connection lost; reconnecting to bucket %q in %v",
+			feed.bucket.Name, retryInterval)
+		err := feed.bucket.Refresh()
+		bucketOK = err == nil
+
+		select {
+		case <-time.After(retryInterval):
+		case <-feed.quit:
+			return
+		}
+		if retryInterval *= 2; retryInterval > maximumRetryInterval {
+			retryInterval = maximumRetryInterval
+		}
+	}
+}
+
+func (feed *TapFeed) connectToNodes() (killSwitch chan bool, err error) {
+	killSwitch = make(chan bool)
+	for _, serverConn := range feed.bucket.getConnPools(false /* not already locked */) {
+		var singleFeed *memcached.TapFeed
+		singleFeed, err = serverConn.StartTapFeed(feed.args)
+		if err != nil {
+			logging.Errorf("go-couchbase: Error connecting to tap feed of %s: %v", serverConn.host, err)
+			feed.closeNodeFeeds()
+			return
+		}
+		feed.nodeFeeds = append(feed.nodeFeeds, singleFeed)
+		go feed.forwardTapEvents(singleFeed, killSwitch, serverConn.host)
+		feed.wg.Add(1)
+	}
+	return
+}
+
+// Goroutine that forwards Tap events from a single node's feed to the aggregate feed.
+func (feed *TapFeed) forwardTapEvents(singleFeed *memcached.TapFeed, killSwitch chan bool, host string) {
+	defer feed.wg.Done()
+	for {
+		select {
+		case event, ok := <-singleFeed.C:
+			if !ok {
+				if singleFeed.Error != nil {
+					logging.Errorf("go-couchbase: Tap feed from %s failed: %v", host, singleFeed.Error)
+				}
+				killSwitch <- true
+				return
+			}
+			feed.output <- event
+		case <-feed.quit:
+			return
+		}
+	}
+}
+
+func (feed *TapFeed) closeNodeFeeds() {
+	for _, f := range feed.nodeFeeds {
+		f.Close()
+	}
+	feed.nodeFeeds = nil
+}
+
+// Close a Tap feed.
+func (feed *TapFeed) Close() error {
+	select {
+	case <-feed.quit:
+		return nil
+	default:
+	}
+
+	feed.closeNodeFeeds()
+	close(feed.quit)
+	feed.wg.Wait()
+	close(feed.output)
+	return nil
+}

vendor/github.com/couchbaselabs/go-couchbase/upr.go

@@ -0,0 +1,398 @@
+package couchbase
+
+import (
+	"log"
+	"sync"
+	"time"
+
+	"fmt"
+	"github.com/couchbase/gomemcached"
+	"github.com/couchbase/gomemcached/client"
+	"github.com/couchbase/goutils/logging"
+)
+
+// A UprFeed streams mutation events from a bucket.
+//
+// Events from the bucket can be read from the channel 'C'.  Remember
+// to call Close() on it when you're done, unless its channel has
+// closed itself already.
+type UprFeed struct {
+	C <-chan *memcached.UprEvent
+
+	bucket          *Bucket
+	nodeFeeds       map[string]*FeedInfo     // The UPR feeds of the individual nodes
+	output          chan *memcached.UprEvent // Same as C but writeably-typed
+	outputClosed    bool
+	quit            chan bool
+	name            string // name of this UPR feed
+	sequence        uint32 // sequence number for this feed
+	connected       bool
+	killSwitch      chan bool
+	closing         bool
+	wg              sync.WaitGroup
+	dcp_buffer_size uint32
+	data_chan_size  int
+}
+
+// UprFeed from a single connection
+type FeedInfo struct {
+	uprFeed   *memcached.UprFeed // UPR feed handle
+	host      string             // hostname
+	connected bool               // connected
+	quit      chan bool          // quit channel
+}
+
+type FailoverLog map[uint16]memcached.FailoverLog
+
+// GetFailoverLogs, get the failover logs for a set of vbucket ids
+func (b *Bucket) GetFailoverLogs(vBuckets []uint16) (FailoverLog, error) {
+
+	// map vbids to their corresponding hosts
+	vbHostList := make(map[string][]uint16)
+	vbm := b.VBServerMap()
+	if len(vbm.VBucketMap) < len(vBuckets) {
+		return nil, fmt.Errorf("vbmap smaller than vbucket list: %v vs. %v",
+			vbm.VBucketMap, vBuckets)
+	}
+
+	for _, vb := range vBuckets {
+		masterID := vbm.VBucketMap[vb][0]
+		master := b.getMasterNode(masterID)
+		if master == "" {
+			return nil, fmt.Errorf("No master found for vb %d", vb)
+		}
+
+		vbList := vbHostList[master]
+		if vbList == nil {
+			vbList = make([]uint16, 0)
+		}
+		vbList = append(vbList, vb)
+		vbHostList[master] = vbList
+	}
+
+	failoverLogMap := make(FailoverLog)
+	for _, serverConn := range b.getConnPools(false /* not already locked */) {
+
+		vbList := vbHostList[serverConn.host]
+		if vbList == nil {
+			continue
+		}
+
+		mc, err := serverConn.Get()
+		if err != nil {
+			logging.Infof("No Free connections for vblist %v", vbList)
+			return nil, fmt.Errorf("No Free connections for host %s",
+				serverConn.host)
+
+		}
+		// close the connection so that it doesn't get reused for upr data
+		// connection
+		defer mc.Close()
+		failoverlogs, err := mc.UprGetFailoverLog(vbList)
+		if err != nil {
+			return nil, fmt.Errorf("Error getting failover log %s host %s",
+				err.Error(), serverConn.host)
+
+		}
+
+		for vb, log := range failoverlogs {
+			failoverLogMap[vb] = *log
+		}
+	}
+
+	return failoverLogMap, nil
+}
+
+func (b *Bucket) StartUprFeed(name string, sequence uint32) (*UprFeed, error) {
+	return b.StartUprFeedWithConfig(name, sequence, 10, DEFAULT_WINDOW_SIZE)
+}
+
+// StartUprFeed creates and starts a new Upr feed
+// No data will be sent on the channel unless vbuckets streams are requested
+func (b *Bucket) StartUprFeedWithConfig(name string, sequence uint32, data_chan_size int, dcp_buffer_size uint32) (*UprFeed, error) {
+
+	feed := &UprFeed{
+		bucket:          b,
+		output:          make(chan *memcached.UprEvent, data_chan_size),
+		quit:            make(chan bool),
+		nodeFeeds:       make(map[string]*FeedInfo, 0),
+		name:            name,
+		sequence:        sequence,
+		killSwitch:      make(chan bool),
+		dcp_buffer_size: dcp_buffer_size,
+		data_chan_size:  data_chan_size,
+	}
+
+	err := feed.connectToNodes()
+	if err != nil {
+		return nil, fmt.Errorf("Cannot connect to bucket %s", err.Error())
+	}
+	feed.connected = true
+	go feed.run()
+
+	feed.C = feed.output
+	return feed, nil
+}
+
+// UprRequestStream starts a stream for a vb on a feed
+func (feed *UprFeed) UprRequestStream(vb uint16, opaque uint16, flags uint32,
+	vuuid, startSequence, endSequence, snapStart, snapEnd uint64) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			log.Panicf("Panic in UprRequestStream. Feed %v Bucket %v", feed, feed.bucket)
+		}
+	}()
+
+	vbm := feed.bucket.VBServerMap()
+	if len(vbm.VBucketMap) < int(vb) {
+		return fmt.Errorf("vbmap smaller than vbucket list: %v vs. %v",
+			vb, vbm.VBucketMap)
+	}
+
+	if int(vb) >= len(vbm.VBucketMap) {
+		return fmt.Errorf("Invalid vbucket id %d", vb)
+	}
+
+	masterID := vbm.VBucketMap[vb][0]
+	master := feed.bucket.getMasterNode(masterID)
+	if master == "" {
+		return fmt.Errorf("Master node not found for vbucket %d", vb)
+	}
+	singleFeed := feed.nodeFeeds[master]
+	if singleFeed == nil {
+		return fmt.Errorf("UprFeed for this host not found")
+	}
+
+	if err := singleFeed.uprFeed.UprRequestStream(vb, opaque, flags,
+		vuuid, startSequence, endSequence, snapStart, snapEnd); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// UprCloseStream ends a vbucket stream.
+func (feed *UprFeed) UprCloseStream(vb, opaqueMSB uint16) error {
+
+	defer func() {
+		if r := recover(); r != nil {
+			log.Panicf("Panic in UprCloseStream. Feed %v Bucket %v ", feed, feed.bucket)
+		}
+	}()
+
+	vbm := feed.bucket.VBServerMap()
+	if len(vbm.VBucketMap) < int(vb) {
+		return fmt.Errorf("vbmap smaller than vbucket list: %v vs. %v",
+			vb, vbm.VBucketMap)
+	}
+
+	if int(vb) >= len(vbm.VBucketMap) {
+		return fmt.Errorf("Invalid vbucket id %d", vb)
+	}
+
+	masterID := vbm.VBucketMap[vb][0]
+	master := feed.bucket.getMasterNode(masterID)
+	if master == "" {
+		return fmt.Errorf("Master node not found for vbucket %d", vb)
+	}
+	singleFeed := feed.nodeFeeds[master]
+	if singleFeed == nil {
+		return fmt.Errorf("UprFeed for this host not found")
+	}
+
+	if err := singleFeed.uprFeed.CloseStream(vb, opaqueMSB); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Goroutine that runs the feed
+func (feed *UprFeed) run() {
+	retryInterval := initialRetryInterval
+	bucketOK := true
+	for {
+		// Connect to the UPR feed of each server node:
+		if bucketOK {
+			// Run until one of the sub-feeds fails:
+			select {
+			case <-feed.killSwitch:
+			case <-feed.quit:
+				return
+			}
+			//feed.closeNodeFeeds()
+			retryInterval = initialRetryInterval
+		}
+
+		if feed.closing == true {
+			// we have been asked to shut down
+			return
+		}
+
+		// On error, try to refresh the bucket in case the list of nodes changed:
+		logging.Infof("go-couchbase: UPR connection lost; reconnecting to bucket %q in %v",
+			feed.bucket.Name, retryInterval)
+
+		if err := feed.bucket.Refresh(); err != nil {
+			// if we fail to refresh the bucket, exit the feed
+			// MB-14917
+			logging.Infof("Unable to refresh bucket %s ", err.Error())
+			close(feed.output)
+			feed.outputClosed = true
+			feed.closeNodeFeeds()
+			return
+		}
+
+		// this will only connect to nodes that are not connected or changed
+		// user will have to reconnect the stream
+		err := feed.connectToNodes()
+		if err != nil {
+			logging.Infof("Unable to connect to nodes..exit ")
+			close(feed.output)
+			feed.outputClosed = true
+			feed.closeNodeFeeds()
+			return
+		}
+		bucketOK = err == nil
+
+		select {
+		case <-time.After(retryInterval):
+		case <-feed.quit:
+			return
+		}
+		if retryInterval *= 2; retryInterval > maximumRetryInterval {
+			retryInterval = maximumRetryInterval
+		}
+	}
+}
+
+func (feed *UprFeed) connectToNodes() (err error) {
+	nodeCount := 0
+	for _, serverConn := range feed.bucket.getConnPools(false /* not already locked */) {
+
+		// this maybe a reconnection, so check if the connection to the node
+		// already exists. Connect only if the node is not found in the list
+		// or connected == false
+		nodeFeed := feed.nodeFeeds[serverConn.host]
+
+		if nodeFeed != nil && nodeFeed.connected == true {
+			continue
+		}
+
+		var singleFeed *memcached.UprFeed
+		var name string
+		if feed.name == "" {
+			name = "DefaultUprClient"
+		} else {
+			name = feed.name
+		}
+		singleFeed, err = serverConn.StartUprFeed(name, feed.sequence, feed.dcp_buffer_size, feed.data_chan_size)
+		if err != nil {
+			logging.Errorf("go-couchbase: Error connecting to upr feed of %s: %v", serverConn.host, err)
+			feed.closeNodeFeeds()
+			return
+		}
+		// add the node to the connection map
+		feedInfo := &FeedInfo{
+			uprFeed:   singleFeed,
+			connected: true,
+			host:      serverConn.host,
+			quit:      make(chan bool),
+		}
+		feed.nodeFeeds[serverConn.host] = feedInfo
+		go feed.forwardUprEvents(feedInfo, feed.killSwitch, serverConn.host)
+		feed.wg.Add(1)
+		nodeCount++
+	}
+	if nodeCount == 0 {
+		return fmt.Errorf("No connection to bucket")
+	}
+
+	return nil
+}
+
+// Goroutine that forwards Upr events from a single node's feed to the aggregate feed.
+func (feed *UprFeed) forwardUprEvents(nodeFeed *FeedInfo, killSwitch chan bool, host string) {
+	singleFeed := nodeFeed.uprFeed
+
+	defer func() {
+		feed.wg.Done()
+		if r := recover(); r != nil {
+			//if feed is not closing, re-throw the panic
+			if feed.outputClosed != true && feed.closing != true {
+				panic(r)
+			} else {
+				logging.Errorf("Panic is recovered. Since feed is closed, exit gracefully")
+
+			}
+		}
+	}()
+
+	for {
+		select {
+		case <-nodeFeed.quit:
+			nodeFeed.connected = false
+			return
+
+		case event, ok := <-singleFeed.C:
+			if !ok {
+				if singleFeed.Error != nil {
+					logging.Errorf("go-couchbase: Upr feed from %s failed: %v", host, singleFeed.Error)
+				}
+				killSwitch <- true
+				return
+			}
+			if feed.outputClosed == true {
+				// someone closed the node feed
+				logging.Infof("Node need closed, returning from forwardUprEvent")
+				return
+			}
+			feed.output <- event
+			if event.Status == gomemcached.NOT_MY_VBUCKET {
+				logging.Infof(" Got a not my vbucket error !! ")
+				if err := feed.bucket.Refresh(); err != nil {
+					logging.Errorf("Unable to refresh bucket %s ", err.Error())
+					feed.closeNodeFeeds()
+					return
+				}
+				// this will only connect to nodes that are not connected or changed
+				// user will have to reconnect the stream
+				if err := feed.connectToNodes(); err != nil {
+					logging.Errorf("Unable to connect to nodes %s", err.Error())
+					return
+				}
+
+			}
+		}
+	}
+}
+
+func (feed *UprFeed) closeNodeFeeds() {
+	for _, f := range feed.nodeFeeds {
+		logging.Infof(" Sending close to forwardUprEvent ")
+		close(f.quit)
+		f.uprFeed.Close()
+	}
+	feed.nodeFeeds = nil
+}
+
+// Close a Upr feed.
+func (feed *UprFeed) Close() error {
+	select {
+	case <-feed.quit:
+		return nil
+	default:
+	}
+
+	feed.closing = true
+	feed.closeNodeFeeds()
+	close(feed.quit)
+
+	feed.wg.Wait()
+	if feed.outputClosed == false {
+		feed.outputClosed = true
+		close(feed.output)
+	}
+
+	return nil
+}

vendor/github.com/couchbaselabs/go-couchbase/users.go

@@ -0,0 +1,119 @@
+package couchbase
+
+import (
+	"bytes"
+	"fmt"
+)
+
+type User struct {
+	Name   string
+	Id     string
+	Domain string
+	Roles  []Role
+}
+
+type Role struct {
+	Role       string
+	BucketName string `json:"bucket_name"`
+}
+
+// Sample:
+// {"role":"admin","name":"Admin","desc":"Can manage ALL cluster features including security.","ce":true}
+// {"role":"query_select","bucket_name":"*","name":"Query Select","desc":"Can execute SELECT statement on bucket to retrieve data"}
+type RoleDescription struct {
+	Role       string
+	Name       string
+	Desc       string
+	Ce         bool
+	BucketName string `json:"bucket_name"`
+}
+
+// Return user-role data, as parsed JSON.
+// Sample:
+//   [{"id":"ivanivanov","name":"Ivan Ivanov","roles":[{"role":"cluster_admin"},{"bucket_name":"default","role":"bucket_admin"}]},
+//    {"id":"petrpetrov","name":"Petr Petrov","roles":[{"role":"replication_admin"}]}]
+func (c *Client) GetUserRoles() ([]interface{}, error) {
+	ret := make([]interface{}, 0, 1)
+	err := c.parseURLResponse("/settings/rbac/users", &ret)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get the configured administrator.
+	// Expected result: {"port":8091,"username":"Administrator"}
+	adminInfo := make(map[string]interface{}, 2)
+	err = c.parseURLResponse("/settings/web", &adminInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create a special entry for the configured administrator.
+	adminResult := map[string]interface{}{
+		"name":   adminInfo["username"],
+		"id":     adminInfo["username"],
+		"domain": "ns_server",
+		"roles": []interface{}{
+			map[string]interface{}{
+				"role": "admin",
+			},
+		},
+	}
+
+	// Add the configured administrator to the list of results.
+	ret = append(ret, adminResult)
+
+	return ret, nil
+}
+
+func (c *Client) GetUserInfoAll() ([]User, error) {
+	ret := make([]User, 0, 16)
+	err := c.parseURLResponse("/settings/rbac/users", &ret)
+	if err != nil {
+		return nil, err
+	}
+	return ret, nil
+}
+
+func rolesToParamFormat(roles []Role) string {
+	var buffer bytes.Buffer
+	for i, role := range roles {
+		if i > 0 {
+			buffer.WriteString(",")
+		}
+		buffer.WriteString(role.Role)
+		if role.BucketName != "" {
+			buffer.WriteString("[")
+			buffer.WriteString(role.BucketName)
+			buffer.WriteString("]")
+		}
+	}
+	return buffer.String()
+}
+
+func (c *Client) PutUserInfo(u *User) error {
+	params := map[string]interface{}{
+		"name":  u.Name,
+		"roles": rolesToParamFormat(u.Roles),
+	}
+	var target string
+	switch u.Domain {
+	case "external":
+		target = "/settings/rbac/users/" + u.Id
+	case "local":
+		target = "/settings/rbac/users/local/" + u.Id
+	default:
+		return fmt.Errorf("Unknown user type: %s", u.Domain)
+	}
+	var ret string // PUT returns an empty string. We ignore it.
+	err := c.parsePutURLResponse(target, params, &ret)
+	return err
+}
+
+func (c *Client) GetRolesAll() ([]RoleDescription, error) {
+	ret := make([]RoleDescription, 0, 32)
+	err := c.parseURLResponse("/settings/rbac/roles", &ret)
+	if err != nil {
+		return nil, err
+	}
+	return ret, nil
+}

vendor/github.com/couchbaselabs/go-couchbase/util.go

@@ -0,0 +1,49 @@
+package couchbase
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+)
+
+// CleanupHost returns the hostname with the given suffix removed.
+func CleanupHost(h, commonSuffix string) string {
+	if strings.HasSuffix(h, commonSuffix) {
+		return h[:len(h)-len(commonSuffix)]
+	}
+	return h
+}
+
+// FindCommonSuffix returns the longest common suffix from the given
+// strings.
+func FindCommonSuffix(input []string) string {
+	rv := ""
+	if len(input) < 2 {
+		return ""
+	}
+	from := input
+	for i := len(input[0]); i > 0; i-- {
+		common := true
+		suffix := input[0][i:]
+		for _, s := range from {
+			if !strings.HasSuffix(s, suffix) {
+				common = false
+				break
+			}
+		}
+		if common {
+			rv = suffix
+		}
+	}
+	return rv
+}
+
+// ParseURL is a wrapper around url.Parse with some sanity-checking
+func ParseURL(urlStr string) (result *url.URL, err error) {
+	result, err = url.Parse(urlStr)
+	if result != nil && result.Scheme == "" {
+		result = nil
+		err = fmt.Errorf("invalid URL <%s>", urlStr)
+	}
+	return
+}

vendor/github.com/couchbaselabs/go-couchbase/vbmap.go

@@ -0,0 +1,77 @@
+package couchbase
+
+var crc32tab = []uint32{
+	0x00000000, 0x77073096, 0xee0e612c, 0x990951ba,
+	0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
+	0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
+	0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
+	0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
+	0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
+	0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,
+	0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
+	0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
+	0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
+	0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940,
+	0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
+	0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116,
+	0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
+	0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
+	0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
+	0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a,
+	0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
+	0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818,
+	0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
+	0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
+	0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
+	0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c,
+	0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
+	0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
+	0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
+	0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
+	0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
+	0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086,
+	0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
+	0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4,
+	0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
+	0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
+	0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
+	0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
+	0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
+	0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe,
+	0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
+	0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
+	0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
+	0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252,
+	0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
+	0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60,
+	0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
+	0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
+	0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
+	0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04,
+	0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
+	0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a,
+	0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
+	0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
+	0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
+	0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e,
+	0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
+	0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
+	0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
+	0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
+	0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
+	0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0,
+	0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
+	0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6,
+	0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
+	0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
+	0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d}
+
+// VBHash finds the vbucket for the given key.
+func (b *Bucket) VBHash(key string) uint32 {
+	crc := uint32(0xffffffff)
+	for x := 0; x < len(key); x++ {
+		crc = (crc >> 8) ^ crc32tab[(uint64(crc)^uint64(key[x]))&0xff]
+	}
+	vbm := b.VBServerMap()
+	return ((^crc) >> 16) & 0x7fff & (uint32(len(vbm.VBucketMap)) - 1)
+}

vendor/github.com/couchbaselabs/go-couchbase/views.go

@@ -0,0 +1,231 @@
+package couchbase
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+// ViewRow represents a single result from a view.
+//
+// Doc is present only if include_docs was set on the request.
+type ViewRow struct {
+	ID    string
+	Key   interface{}
+	Value interface{}
+	Doc   *interface{}
+}
+
+// A ViewError is a node-specific error indicating a partial failure
+// within a view result.
+type ViewError struct {
+	From   string
+	Reason string
+}
+
+func (ve ViewError) Error() string {
+	return "Node: " + ve.From + ", reason: " + ve.Reason
+}
+
+// ViewResult holds the entire result set from a view request,
+// including the rows and the errors.
+type ViewResult struct {
+	TotalRows int `json:"total_rows"`
+	Rows      []ViewRow
+	Errors    []ViewError
+}
+
+func (b *Bucket) randomBaseURL() (*url.URL, error) {
+	nodes := b.HealthyNodes()
+	if len(nodes) == 0 {
+		return nil, errors.New("no available couch rest URLs")
+	}
+	nodeNo := rand.Intn(len(nodes))
+	node := nodes[nodeNo]
+
+	b.RLock()
+	name := b.Name
+	pool := b.pool
+	b.RUnlock()
+
+	u, err := ParseURL(node.CouchAPIBase)
+	if err != nil {
+		return nil, fmt.Errorf("config error: Bucket %q node #%d CouchAPIBase=%q: %v",
+			name, nodeNo, node.CouchAPIBase, err)
+	} else if pool != nil {
+		u.User = pool.client.BaseURL.User
+	}
+	return u, err
+}
+
+const START_NODE_ID = -1
+
+func (b *Bucket) randomNextURL(lastNode int) (*url.URL, int, error) {
+	nodes := b.HealthyNodes()
+	if len(nodes) == 0 {
+		return nil, -1, errors.New("no available couch rest URLs")
+	}
+
+	var nodeNo int
+	if lastNode == START_NODE_ID || lastNode >= len(nodes) {
+		// randomly select a node if the value of lastNode is invalid
+		nodeNo = rand.Intn(len(nodes))
+	} else {
+		// wrap around the node list
+		nodeNo = (lastNode + 1) % len(nodes)
+	}
+
+	b.RLock()
+	name := b.Name
+	pool := b.pool
+	b.RUnlock()
+
+	node := nodes[nodeNo]
+	u, err := ParseURL(node.CouchAPIBase)
+	if err != nil {
+		return nil, -1, fmt.Errorf("config error: Bucket %q node #%d CouchAPIBase=%q: %v",
+			name, nodeNo, node.CouchAPIBase, err)
+	} else if pool != nil {
+		u.User = pool.client.BaseURL.User
+	}
+	return u, nodeNo, err
+}
+
+// DocID is the document ID type for the startkey_docid parameter in
+// views.
+type DocID string
+
+func qParam(k, v string) string {
+	format := `"%s"`
+	switch k {
+	case "startkey_docid", "endkey_docid", "stale":
+		format = "%s"
+	}
+	return fmt.Sprintf(format, v)
+}
+
+// ViewURL constructs a URL for a view with the given ddoc, view name,
+// and parameters.
+func (b *Bucket) ViewURL(ddoc, name string,
+	params map[string]interface{}) (string, error) {
+	u, err := b.randomBaseURL()
+	if err != nil {
+		return "", err
+	}
+
+	values := url.Values{}
+	for k, v := range params {
+		switch t := v.(type) {
+		case DocID:
+			values[k] = []string{string(t)}
+		case string:
+			values[k] = []string{qParam(k, t)}
+		case int:
+			values[k] = []string{fmt.Sprintf(`%d`, t)}
+		case bool:
+			values[k] = []string{fmt.Sprintf(`%v`, t)}
+		default:
+			b, err := json.Marshal(v)
+			if err != nil {
+				return "", fmt.Errorf("unsupported value-type %T in Query, "+
+					"json encoder said %v", t, err)
+			}
+			values[k] = []string{fmt.Sprintf(`%v`, string(b))}
+		}
+	}
+
+	if ddoc == "" && name == "_all_docs" {
+		u.Path = fmt.Sprintf("/%s/_all_docs", b.GetName())
+	} else {
+		u.Path = fmt.Sprintf("/%s/_design/%s/_view/%s", b.GetName(), ddoc, name)
+	}
+	u.RawQuery = values.Encode()
+
+	return u.String(), nil
+}
+
+// ViewCallback is called for each view invocation.
+var ViewCallback func(ddoc, name string, start time.Time, err error)
+
+// ViewCustom performs a view request that can map row values to a
+// custom type.
+//
+// See the source to View for an example usage.
+func (b *Bucket) ViewCustom(ddoc, name string, params map[string]interface{},
+	vres interface{}) (err error) {
+	if SlowServerCallWarningThreshold > 0 {
+		defer slowLog(time.Now(), "call to ViewCustom(%q, %q)", ddoc, name)
+	}
+
+	if ViewCallback != nil {
+		defer func(t time.Time) { ViewCallback(ddoc, name, t, err) }(time.Now())
+	}
+
+	u, err := b.ViewURL(ddoc, name, params)
+	if err != nil {
+		return err
+	}
+
+	req, err := http.NewRequest("GET", u, nil)
+	if err != nil {
+		return err
+	}
+
+	ah := b.authHandler(false /* bucket not yet locked */)
+	maybeAddAuth(req, ah)
+
+	res, err := doHTTPRequest(req)
+	if err != nil {
+		return fmt.Errorf("error starting view req at %v: %v", u, err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != 200 {
+		bod := make([]byte, 512)
+		l, _ := res.Body.Read(bod)
+		return fmt.Errorf("error executing view req at %v: %v - %s",
+			u, res.Status, bod[:l])
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err := json.Unmarshal(body, vres); err != nil {
+		return nil
+	}
+
+	return nil
+}
+
+// View executes a view.
+//
+// The ddoc parameter is just the bare name of your design doc without
+// the "_design/" prefix.
+//
+// Parameters are string keys with values that correspond to couchbase
+// view parameters.  Primitive should work fairly naturally (booleans,
+// ints, strings, etc...) and other values will attempt to be JSON
+// marshaled (useful for array indexing on on view keys, for example).
+//
+// Example:
+//
+//   res, err := couchbase.View("myddoc", "myview", map[string]interface{}{
+//       "group_level": 2,
+//       "startkey_docid":    []interface{}{"thing"},
+//       "endkey_docid":      []interface{}{"thing", map[string]string{}},
+//       "stale": false,
+//       })
+func (b *Bucket) View(ddoc, name string, params map[string]interface{}) (ViewResult, error) {
+	vres := ViewResult{}
+
+	if err := b.ViewCustom(ddoc, name, params, &vres); err != nil {
+		//error in accessing views. Retry once after a bucket refresh
+		b.Refresh()
+		return vres, b.ViewCustom(ddoc, name, params, &vres)
+	} else {
+		return vres, nil
+	}
+}

vendor/github.com/go-macaron/session/couchbase/couchbase.go

@@ -0,0 +1,228 @@
+// Copyright 2013 Beego Authors
+// Copyright 2014 The Macaron Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package session
+
+import (
+	"strings"
+	"sync"
+
+	"github.com/couchbaselabs/go-couchbase"
+
+	"github.com/go-macaron/session"
+)
+
+// CouchbaseSessionStore represents a couchbase session store implementation.
+type CouchbaseSessionStore struct {
+	b           *couchbase.Bucket
+	sid         string
+	lock        sync.RWMutex
+	data        map[interface{}]interface{}
+	maxlifetime int64
+}
+
+// Set sets value to given key in session.
+func (s *CouchbaseSessionStore) Set(key, val interface{}) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.data[key] = val
+	return nil
+}
+
+// Get gets value by given key in session.
+func (s *CouchbaseSessionStore) Get(key interface{}) interface{} {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+
+	return s.data[key]
+}
+
+// Delete delete a key from session.
+func (s *CouchbaseSessionStore) Delete(key interface{}) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	delete(s.data, key)
+	return nil
+}
+
+// ID returns current session ID.
+func (s *CouchbaseSessionStore) ID() string {
+	return s.sid
+}
+
+// Release releases resource and save data to provider.
+func (s *CouchbaseSessionStore) Release() error {
+	defer s.b.Close()
+
+	// Skip encoding if the data is empty
+	if len(s.data) == 0 {
+		return nil
+	}
+
+	data, err := session.EncodeGob(s.data)
+	if err != nil {
+		return err
+	}
+
+	return s.b.Set(s.sid, int(s.maxlifetime), data)
+}
+
+// Flush deletes all session data.
+func (s *CouchbaseSessionStore) Flush() error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.data = make(map[interface{}]interface{})
+	return nil
+}
+
+// CouchbaseProvider represents a couchbase session provider implementation.
+type CouchbaseProvider struct {
+	maxlifetime int64
+	connStr     string
+	pool        string
+	bucket      string
+	b           *couchbase.Bucket
+}
+
+func (cp *CouchbaseProvider) getBucket() *couchbase.Bucket {
+	c, err := couchbase.Connect(cp.connStr)
+	if err != nil {
+		return nil
+	}
+
+	pool, err := c.GetPool(cp.pool)
+	if err != nil {
+		return nil
+	}
+
+	bucket, err := pool.GetBucket(cp.bucket)
+	if err != nil {
+		return nil
+	}
+
+	return bucket
+}
+
+// Init initializes memory session provider.
+// connStr is couchbase server REST/JSON URL
+// e.g. http://host:port/, Pool, Bucket
+func (p *CouchbaseProvider) Init(maxlifetime int64, connStr string) error {
+	p.maxlifetime = maxlifetime
+	configs := strings.Split(connStr, ",")
+	if len(configs) > 0 {
+		p.connStr = configs[0]
+	}
+	if len(configs) > 1 {
+		p.pool = configs[1]
+	}
+	if len(configs) > 2 {
+		p.bucket = configs[2]
+	}
+
+	return nil
+}
+
+// Read returns raw session store by session ID.
+func (p *CouchbaseProvider) Read(sid string) (session.RawStore, error) {
+	p.b = p.getBucket()
+
+	var doc []byte
+
+	err := p.b.Get(sid, &doc)
+	var kv map[interface{}]interface{}
+	if doc == nil {
+		kv = make(map[interface{}]interface{})
+	} else {
+		kv, err = session.DecodeGob(doc)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	cs := &CouchbaseSessionStore{b: p.b, sid: sid, data: kv, maxlifetime: p.maxlifetime}
+	return cs, nil
+}
+
+// Exist returns true if session with given ID exists.
+func (p *CouchbaseProvider) Exist(sid string) bool {
+	p.b = p.getBucket()
+	defer p.b.Close()
+
+	var doc []byte
+
+	if err := p.b.Get(sid, &doc); err != nil || doc == nil {
+		return false
+	} else {
+		return true
+	}
+}
+
+// Destory deletes a session by session ID.
+func (p *CouchbaseProvider) Destory(sid string) error {
+	p.b = p.getBucket()
+	defer p.b.Close()
+
+	p.b.Delete(sid)
+	return nil
+}
+
+// Regenerate regenerates a session store from old session ID to new one.
+func (p *CouchbaseProvider) Regenerate(oldsid, sid string) (session.RawStore, error) {
+	p.b = p.getBucket()
+
+	var doc []byte
+	if err := p.b.Get(oldsid, &doc); err != nil || doc == nil {
+		p.b.Set(sid, int(p.maxlifetime), "")
+	} else {
+		err := p.b.Delete(oldsid)
+		if err != nil {
+			return nil, err
+		}
+		_, _ = p.b.Add(sid, int(p.maxlifetime), doc)
+	}
+
+	err := p.b.Get(sid, &doc)
+	if err != nil {
+		return nil, err
+	}
+	var kv map[interface{}]interface{}
+	if doc == nil {
+		kv = make(map[interface{}]interface{})
+	} else {
+		kv, err = session.DecodeGob(doc)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	cs := &CouchbaseSessionStore{b: p.b, sid: sid, data: kv, maxlifetime: p.maxlifetime}
+	return cs, nil
+}
+
+// Count counts and returns number of sessions.
+func (p *CouchbaseProvider) Count() int {
+	// FIXME
+	return 0
+}
+
+// GC calls GC to clean expired sessions.
+func (p *CouchbaseProvider) GC() {}
+
+func init() {
+	session.Register("couchbase", &CouchbaseProvider{})
+}

vendor/github.com/go-macaron/session/file.go

@@ -81,6 +81,11 @@ func (s *FileStore) Release() error {
 	s.p.lock.Lock()
 	defer s.p.lock.Unlock()
 
+	// Skip encoding if the data is empty
+	if len(s.data) == 0 {
+		return nil
+	}
+
 	data, err := EncodeGob(s.data)
 	if err != nil {
 		return err

vendor/github.com/go-macaron/session/flash.go

@@ -0,0 +1,61 @@
+// Copyright 2018 The Macaron Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package session
+
+import (
+	"net/url"
+
+	"gopkg.in/macaron.v1"
+)
+
+type Flash struct {
+	ctx *macaron.Context
+	url.Values
+	ErrorMsg, WarningMsg, InfoMsg, SuccessMsg string
+}
+
+func (f *Flash) set(name, msg string, current ...bool) {
+	isShow := false
+	if (len(current) == 0 && macaron.FlashNow) ||
+		(len(current) > 0 && current[0]) {
+		isShow = true
+	}
+
+	if isShow {
+		f.ctx.Data["Flash"] = f
+	} else {
+		f.Set(name, msg)
+	}
+}
+
+func (f *Flash) Error(msg string, current ...bool) {
+	f.ErrorMsg = msg
+	f.set("error", msg, current...)
+}
+
+func (f *Flash) Warning(msg string, current ...bool) {
+	f.WarningMsg = msg
+	f.set("warning", msg, current...)
+}
+
+func (f *Flash) Info(msg string, current ...bool) {
+	f.InfoMsg = msg
+	f.set("info", msg, current...)
+}
+
+func (f *Flash) Success(msg string, current ...bool) {
+	f.SuccessMsg = msg
+	f.set("success", msg, current...)
+}

vendor/github.com/go-macaron/session/memcache/memcache.go

@@ -0,0 +1,204 @@
+// Copyright 2013 Beego Authors
+// Copyright 2014 The Macaron Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package session
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/bradfitz/gomemcache/memcache"
+
+	"github.com/go-macaron/session"
+)
+
+// MemcacheStore represents a memcache session store implementation.
+type MemcacheStore struct {
+	c      *memcache.Client
+	sid    string
+	expire int32
+	lock   sync.RWMutex
+	data   map[interface{}]interface{}
+}
+
+// NewMemcacheStore creates and returns a memcache session store.
+func NewMemcacheStore(c *memcache.Client, sid string, expire int32, kv map[interface{}]interface{}) *MemcacheStore {
+	return &MemcacheStore{
+		c:      c,
+		sid:    sid,
+		expire: expire,
+		data:   kv,
+	}
+}
+
+func NewItem(sid string, data []byte, expire int32) *memcache.Item {
+	return &memcache.Item{
+		Key:        sid,
+		Value:      data,
+		Expiration: expire,
+	}
+}
+
+// Set sets value to given key in session.
+func (s *MemcacheStore) Set(key, val interface{}) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.data[key] = val
+	return nil
+}
+
+// Get gets value by given key in session.
+func (s *MemcacheStore) Get(key interface{}) interface{} {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+
+	return s.data[key]
+}
+
+// Delete delete a key from session.
+func (s *MemcacheStore) Delete(key interface{}) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	delete(s.data, key)
+	return nil
+}
+
+// ID returns current session ID.
+func (s *MemcacheStore) ID() string {
+	return s.sid
+}
+
+// Release releases resource and save data to provider.
+func (s *MemcacheStore) Release() error {
+	// Skip encoding if the data is empty
+	if len(s.data) == 0 {
+		return nil
+	}
+
+	data, err := session.EncodeGob(s.data)
+	if err != nil {
+		return err
+	}
+
+	return s.c.Set(NewItem(s.sid, data, s.expire))
+}
+
+// Flush deletes all session data.
+func (s *MemcacheStore) Flush() error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.data = make(map[interface{}]interface{})
+	return nil
+}
+
+// MemcacheProvider represents a memcache session provider implementation.
+type MemcacheProvider struct {
+	c      *memcache.Client
+	expire int32
+}
+
+// Init initializes memcache session provider.
+// connStrs: 127.0.0.1:9090;127.0.0.1:9091
+func (p *MemcacheProvider) Init(expire int64, connStrs string) error {
+	p.expire = int32(expire)
+	p.c = memcache.New(strings.Split(connStrs, ";")...)
+	return nil
+}
+
+// Read returns raw session store by session ID.
+func (p *MemcacheProvider) Read(sid string) (session.RawStore, error) {
+	if !p.Exist(sid) {
+		if err := p.c.Set(NewItem(sid, []byte(""), p.expire)); err != nil {
+			return nil, err
+		}
+	}
+
+	var kv map[interface{}]interface{}
+	item, err := p.c.Get(sid)
+	if err != nil {
+		return nil, err
+	}
+	if len(item.Value) == 0 {
+		kv = make(map[interface{}]interface{})
+	} else {
+		kv, err = session.DecodeGob(item.Value)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return NewMemcacheStore(p.c, sid, p.expire, kv), nil
+}
+
+// Exist returns true if session with given ID exists.
+func (p *MemcacheProvider) Exist(sid string) bool {
+	_, err := p.c.Get(sid)
+	return err == nil
+}
+
+// Destory deletes a session by session ID.
+func (p *MemcacheProvider) Destory(sid string) error {
+	return p.c.Delete(sid)
+}
+
+// Regenerate regenerates a session store from old session ID to new one.
+func (p *MemcacheProvider) Regenerate(oldsid, sid string) (_ session.RawStore, err error) {
+	if p.Exist(sid) {
+		return nil, fmt.Errorf("new sid '%s' already exists", sid)
+	}
+
+	item := NewItem(sid, []byte(""), p.expire)
+	if p.Exist(oldsid) {
+		item, err = p.c.Get(oldsid)
+		if err != nil {
+			return nil, err
+		} else if err = p.c.Delete(oldsid); err != nil {
+			return nil, err
+		}
+		item.Key = sid
+	}
+	if err = p.c.Set(item); err != nil {
+		return nil, err
+	}
+
+	var kv map[interface{}]interface{}
+	if len(item.Value) == 0 {
+		kv = make(map[interface{}]interface{})
+	} else {
+		kv, err = session.DecodeGob(item.Value)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return NewMemcacheStore(p.c, sid, p.expire, kv), nil
+}
+
+// Count counts and returns number of sessions.
+func (p *MemcacheProvider) Count() int {
+	// FIXME: how come this library does not have Stats method?
+	return -1
+}
+
+// GC calls GC to clean expired sessions.
+func (p *MemcacheProvider) GC() {}
+
+func init() {
+	session.Register("memcache", &MemcacheProvider{})
+}

vendor/github.com/go-macaron/session/mysql/mysql.go

@@ -0,0 +1,200 @@
+// Copyright 2013 Beego Authors
+// Copyright 2014 The Macaron Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package session
+
+import (
+	"database/sql"
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	_ "github.com/go-sql-driver/mysql"
+
+	"github.com/go-macaron/session"
+)
+
+// MysqlStore represents a mysql session store implementation.
+type MysqlStore struct {
+	c    *sql.DB
+	sid  string
+	lock sync.RWMutex
+	data map[interface{}]interface{}
+}
+
+// NewMysqlStore creates and returns a mysql session store.
+func NewMysqlStore(c *sql.DB, sid string, kv map[interface{}]interface{}) *MysqlStore {
+	return &MysqlStore{
+		c:    c,
+		sid:  sid,
+		data: kv,
+	}
+}
+
+// Set sets value to given key in session.
+func (s *MysqlStore) Set(key, val interface{}) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.data[key] = val
+	return nil
+}
+
+// Get gets value by given key in session.
+func (s *MysqlStore) Get(key interface{}) interface{} {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+
+	return s.data[key]
+}
+
+// Delete delete a key from session.
+func (s *MysqlStore) Delete(key interface{}) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	delete(s.data, key)
+	return nil
+}
+
+// ID returns current session ID.
+func (s *MysqlStore) ID() string {
+	return s.sid
+}
+
+// Release releases resource and save data to provider.
+func (s *MysqlStore) Release() error {
+	// Skip encoding if the data is empty
+	if len(s.data) == 0 {
+		return nil
+	}
+
+	data, err := session.EncodeGob(s.data)
+	if err != nil {
+		return err
+	}
+
+	_, err = s.c.Exec("UPDATE session SET data=?, expiry=? WHERE `key`=?",
+		data, time.Now().Unix(), s.sid)
+	return err
+}
+
+// Flush deletes all session data.
+func (s *MysqlStore) Flush() error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.data = make(map[interface{}]interface{})
+	return nil
+}
+
+// MysqlProvider represents a mysql session provider implementation.
+type MysqlProvider struct {
+	c      *sql.DB
+	expire int64
+}
+
+// Init initializes mysql session provider.
+// connStr: username:password@protocol(address)/dbname?param=value
+func (p *MysqlProvider) Init(expire int64, connStr string) (err error) {
+	p.expire = expire
+
+	p.c, err = sql.Open("mysql", connStr)
+	if err != nil {
+		return err
+	}
+	return p.c.Ping()
+}
+
+// Read returns raw session store by session ID.
+func (p *MysqlProvider) Read(sid string) (session.RawStore, error) {
+	var data []byte
+	err := p.c.QueryRow("SELECT data FROM session WHERE `key`=?", sid).Scan(&data)
+	if err == sql.ErrNoRows {
+		_, err = p.c.Exec("INSERT INTO session(`key`,data,expiry) VALUES(?,?,?)",
+			sid, "", time.Now().Unix())
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var kv map[interface{}]interface{}
+	if len(data) == 0 {
+		kv = make(map[interface{}]interface{})
+	} else {
+		kv, err = session.DecodeGob(data)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return NewMysqlStore(p.c, sid, kv), nil
+}
+
+// Exist returns true if session with given ID exists.
+func (p *MysqlProvider) Exist(sid string) bool {
+	var data []byte
+	err := p.c.QueryRow("SELECT data FROM session WHERE `key`=?", sid).Scan(&data)
+	if err != nil && err != sql.ErrNoRows {
+		panic("session/mysql: error checking existence: " + err.Error())
+	}
+	return err != sql.ErrNoRows
+}
+
+// Destory deletes a session by session ID.
+func (p *MysqlProvider) Destory(sid string) error {
+	_, err := p.c.Exec("DELETE FROM session WHERE `key`=?", sid)
+	return err
+}
+
+// Regenerate regenerates a session store from old session ID to new one.
+func (p *MysqlProvider) Regenerate(oldsid, sid string) (_ session.RawStore, err error) {
+	if p.Exist(sid) {
+		return nil, fmt.Errorf("new sid '%s' already exists", sid)
+	}
+
+	if !p.Exist(oldsid) {
+		if _, err = p.c.Exec("INSERT INTO session(`key`,data,expiry) VALUES(?,?,?)",
+			oldsid, "", time.Now().Unix()); err != nil {
+			return nil, err
+		}
+	}
+
+	if _, err = p.c.Exec("UPDATE session SET `key`=? WHERE `key`=?", sid, oldsid); err != nil {
+		return nil, err
+	}
+
+	return p.Read(sid)
+}
+
+// Count counts and returns number of sessions.
+func (p *MysqlProvider) Count() (total int) {
+	if err := p.c.QueryRow("SELECT COUNT(*) AS NUM FROM session").Scan(&total); err != nil {
+		panic("session/mysql: error counting records: " + err.Error())
+	}
+	return total
+}
+
+// GC calls GC to clean expired sessions.
+func (p *MysqlProvider) GC() {
+	if _, err := p.c.Exec("DELETE FROM session WHERE  expiry + ? <= UNIX_TIMESTAMP(NOW())", p.expire); err != nil {
+		log.Printf("session/mysql: error garbage collecting: %v", err)
+	}
+}
+
+func init() {
+	session.Register("mysql", &MysqlProvider{})
+}

vendor/github.com/go-macaron/session/nodb/nodb.go

@@ -0,0 +1,208 @@
+// Copyright 2015 The Macaron Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package session
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/lunny/nodb"
+	"github.com/lunny/nodb/config"
+
+	"github.com/go-macaron/session"
+)
+
+// NodbStore represents a nodb session store implementation.
+type NodbStore struct {
+	c      *nodb.DB
+	sid    string
+	expire int64
+	lock   sync.RWMutex
+	data   map[interface{}]interface{}
+}
+
+// NewNodbStore creates and returns a ledis session store.
+func NewNodbStore(c *nodb.DB, sid string, expire int64, kv map[interface{}]interface{}) *NodbStore {
+	return &NodbStore{
+		c:      c,
+		expire: expire,
+		sid:    sid,
+		data:   kv,
+	}
+}
+
+// Set sets value to given key in session.
+func (s *NodbStore) Set(key, val interface{}) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.data[key] = val
+	return nil
+}
+
+// Get gets value by given key in session.
+func (s *NodbStore) Get(key interface{}) interface{} {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+
+	return s.data[key]
+}
+
+// Delete delete a key from session.
+func (s *NodbStore) Delete(key interface{}) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	delete(s.data, key)
+	return nil
+}
+
+// ID returns current session ID.
+func (s *NodbStore) ID() string {
+	return s.sid
+}
+
+// Release releases resource and save data to provider.
+func (s *NodbStore) Release() error {
+	// Skip encoding if the data is empty
+	if len(s.data) == 0 {
+		return nil
+	}
+
+	data, err := session.EncodeGob(s.data)
+	if err != nil {
+		return err
+	}
+
+	if err = s.c.Set([]byte(s.sid), data); err != nil {
+		return err
+	}
+	_, err = s.c.Expire([]byte(s.sid), s.expire)
+	return err
+}
+
+// Flush deletes all session data.
+func (s *NodbStore) Flush() error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.data = make(map[interface{}]interface{})
+	return nil
+}
+
+// NodbProvider represents a ledis session provider implementation.
+type NodbProvider struct {
+	c      *nodb.DB
+	expire int64
+}
+
+// Init initializes nodb session provider.
+func (p *NodbProvider) Init(expire int64, configs string) error {
+	p.expire = expire
+
+	cfg := new(config.Config)
+	cfg.DataDir = configs
+	dbs, err := nodb.Open(cfg)
+	if err != nil {
+		return fmt.Errorf("session/nodb: error opening db: %v", err)
+	}
+
+	p.c, err = dbs.Select(0)
+	return err
+}
+
+// Read returns raw session store by session ID.
+func (p *NodbProvider) Read(sid string) (session.RawStore, error) {
+	if !p.Exist(sid) {
+		if err := p.c.Set([]byte(sid), []byte("")); err != nil {
+			return nil, err
+		}
+	}
+
+	var kv map[interface{}]interface{}
+	kvs, err := p.c.Get([]byte(sid))
+	if err != nil {
+		return nil, err
+	}
+	if len(kvs) == 0 {
+		kv = make(map[interface{}]interface{})
+	} else {
+		kv, err = session.DecodeGob(kvs)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return NewNodbStore(p.c, sid, p.expire, kv), nil
+}
+
+// Exist returns true if session with given ID exists.
+func (p *NodbProvider) Exist(sid string) bool {
+	count, err := p.c.Exists([]byte(sid))
+	return err == nil && count > 0
+}
+
+// Destory deletes a session by session ID.
+func (p *NodbProvider) Destory(sid string) error {
+	_, err := p.c.Del([]byte(sid))
+	return err
+}
+
+// Regenerate regenerates a session store from old session ID to new one.
+func (p *NodbProvider) Regenerate(oldsid, sid string) (_ session.RawStore, err error) {
+	if p.Exist(sid) {
+		return nil, fmt.Errorf("new sid '%s' already exists", sid)
+	}
+
+	kvs := make([]byte, 0)
+	if p.Exist(oldsid) {
+		if kvs, err = p.c.Get([]byte(oldsid)); err != nil {
+			return nil, err
+		} else if _, err = p.c.Del([]byte(oldsid)); err != nil {
+			return nil, err
+		}
+	}
+
+	if err = p.c.Set([]byte(sid), kvs); err != nil {
+		return nil, err
+	} else if _, err = p.c.Expire([]byte(sid), p.expire); err != nil {
+		return nil, err
+	}
+
+	var kv map[interface{}]interface{}
+	if len(kvs) == 0 {
+		kv = make(map[interface{}]interface{})
+	} else {
+		kv, err = session.DecodeGob([]byte(kvs))
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return NewNodbStore(p.c, sid, p.expire, kv), nil
+}
+
+// Count counts and returns number of sessions.
+func (p *NodbProvider) Count() int {
+	// FIXME: how come this library does not have DbSize() method?
+	return -1
+}
+
+// GC calls GC to clean expired sessions.
+func (p *NodbProvider) GC() {}
+
+func init() {
+	session.Register("nodb", &NodbProvider{})
+}

vendor/github.com/go-macaron/session/postgres/postgres.go

@@ -0,0 +1,201 @@
+// Copyright 2013 Beego Authors
+// Copyright 2014 The Macaron Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package session
+
+import (
+	"database/sql"
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	_ "github.com/lib/pq"
+
+	"github.com/go-macaron/session"
+)
+
+// PostgresStore represents a postgres session store implementation.
+type PostgresStore struct {
+	c    *sql.DB
+	sid  string
+	lock sync.RWMutex
+	data map[interface{}]interface{}
+}
+
+// NewPostgresStore creates and returns a postgres session store.
+func NewPostgresStore(c *sql.DB, sid string, kv map[interface{}]interface{}) *PostgresStore {
+	return &PostgresStore{
+		c:    c,
+		sid:  sid,
+		data: kv,
+	}
+}
+
+// Set sets value to given key in session.
+func (s *PostgresStore) Set(key, value interface{}) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.data[key] = value
+	return nil
+}
+
+// Get gets value by given key in session.
+func (s *PostgresStore) Get(key interface{}) interface{} {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+
+	return s.data[key]
+}
+
+// Delete delete a key from session.
+func (s *PostgresStore) Delete(key interface{}) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	delete(s.data, key)
+	return nil
+}
+
+// ID returns current session ID.
+func (s *PostgresStore) ID() string {
+	return s.sid
+}
+
+// save postgres session values to database.
+// must call this method to save values to database.
+func (s *PostgresStore) Release() error {
+	// Skip encoding if the data is empty
+	if len(s.data) == 0 {
+		return nil
+	}
+
+	data, err := session.EncodeGob(s.data)
+	if err != nil {
+		return err
+	}
+
+	_, err = s.c.Exec("UPDATE session SET data=$1, expiry=$2 WHERE key=$3",
+		data, time.Now().Unix(), s.sid)
+	return err
+}
+
+// Flush deletes all session data.
+func (s *PostgresStore) Flush() error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.data = make(map[interface{}]interface{})
+	return nil
+}
+
+// PostgresProvider represents a postgres session provider implementation.
+type PostgresProvider struct {
+	c           *sql.DB
+	maxlifetime int64
+}
+
+// Init initializes postgres session provider.
+// connStr: user=a password=b host=localhost port=5432 dbname=c sslmode=disable
+func (p *PostgresProvider) Init(maxlifetime int64, connStr string) (err error) {
+	p.maxlifetime = maxlifetime
+
+	p.c, err = sql.Open("postgres", connStr)
+	if err != nil {
+		return err
+	}
+	return p.c.Ping()
+}
+
+// Read returns raw session store by session ID.
+func (p *PostgresProvider) Read(sid string) (session.RawStore, error) {
+	var data []byte
+	err := p.c.QueryRow("SELECT data FROM session WHERE key=$1", sid).Scan(&data)
+	if err == sql.ErrNoRows {
+		_, err = p.c.Exec("INSERT INTO session(key,data,expiry) VALUES($1,$2,$3)",
+			sid, "", time.Now().Unix())
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var kv map[interface{}]interface{}
+	if len(data) == 0 {
+		kv = make(map[interface{}]interface{})
+	} else {
+		kv, err = session.DecodeGob(data)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return NewPostgresStore(p.c, sid, kv), nil
+}
+
+// Exist returns true if session with given ID exists.
+func (p *PostgresProvider) Exist(sid string) bool {
+	var data []byte
+	err := p.c.QueryRow("SELECT data FROM session WHERE key=$1", sid).Scan(&data)
+	if err != nil && err != sql.ErrNoRows {
+		panic("session/postgres: error checking existence: " + err.Error())
+	}
+	return err != sql.ErrNoRows
+}
+
+// Destory deletes a session by session ID.
+func (p *PostgresProvider) Destory(sid string) error {
+	_, err := p.c.Exec("DELETE FROM session WHERE key=$1", sid)
+	return err
+}
+
+// Regenerate regenerates a session store from old session ID to new one.
+func (p *PostgresProvider) Regenerate(oldsid, sid string) (_ session.RawStore, err error) {
+	if p.Exist(sid) {
+		return nil, fmt.Errorf("new sid '%s' already exists", sid)
+	}
+
+	if !p.Exist(oldsid) {
+		if _, err = p.c.Exec("INSERT INTO session(key,data,expiry) VALUES($1,$2,$3)",
+			oldsid, "", time.Now().Unix()); err != nil {
+			return nil, err
+		}
+	}
+
+	if _, err = p.c.Exec("UPDATE session SET key=$1 WHERE key=$2", sid, oldsid); err != nil {
+		return nil, err
+	}
+
+	return p.Read(sid)
+}
+
+// Count counts and returns number of sessions.
+func (p *PostgresProvider) Count() (total int) {
+	if err := p.c.QueryRow("SELECT COUNT(*) AS NUM FROM session").Scan(&total); err != nil {
+		panic("session/postgres: error counting records: " + err.Error())
+	}
+	return total
+}
+
+// GC calls GC to clean expired sessions.
+func (p *PostgresProvider) GC() {
+	if _, err := p.c.Exec("DELETE FROM session WHERE EXTRACT(EPOCH FROM NOW()) - expiry > $1", p.maxlifetime); err != nil {
+		log.Printf("session/postgres: error garbage collecting: %v", err)
+	}
+}
+
+func init() {
+	session.Register("postgres", &PostgresProvider{})
+}

vendor/github.com/go-macaron/session/redis/redis.go

@@ -81,6 +81,11 @@ func (s *RedisStore) ID() string {
 
 // Release releases resource and save data to provider.
 func (s *RedisStore) Release() error {
+	// Skip encoding if the data is empty
+	if len(s.data) == 0 {
+		return nil
+	}
+
 	data, err := session.EncodeGob(s.data)
 	if err != nil {
 		return err
@@ -153,7 +158,7 @@ func (p *RedisProvider) Init(maxlifetime int64, configs string) (err error) {
 func (p *RedisProvider) Read(sid string) (session.RawStore, error) {
 	psid := p.prefix + sid
 	if !p.Exist(sid) {
-		if err := p.c.Set(psid, "").Err(); err != nil {
+		if err := p.c.SetEx(psid, p.duration, "").Err(); err != nil {
 			return nil, err
 		}
 	}

vendor/github.com/go-macaron/session/session.go

@@ -22,13 +22,12 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
-	"strings"
 	"time"
 
 	"gopkg.in/macaron.v1"
 )
 
-const _VERSION = "0.4.0"
+const _VERSION = "0.6.0"
 
 func Version() string {
 	return _VERSION
@@ -96,6 +95,8 @@ type Options struct {
 	IDLength int
 	// Configuration section name. Default is "session".
 	Section string
+	// Ignore release for websocket. Default is false.
+	IgnoreReleaseForWebSocket bool
 }
 
 func prepareOptions(options []Options) Options {
@@ -138,6 +139,9 @@ func prepareOptions(options []Options) Options {
 	if opt.IDLength == 0 {
 		opt.IDLength = sec.Key("ID_LENGTH").MustInt(16)
 	}
+	if !opt.IgnoreReleaseForWebSocket {
+		opt.IgnoreReleaseForWebSocket = sec.Key("IGNORE_RELEASE_FOR_WEBSOCKET").MustBool()
+	}
 
 	return opt
 }
@@ -187,6 +191,10 @@ func Sessioner(options ...Options) macaron.Handler {
 
 		ctx.Next()
 
+		if manager.opt.IgnoreReleaseForWebSocket && ctx.Req.Header.Get("Upgrade") == "websocket" {
+			return
+		}
+
 		if err = sess.Release(); err != nil {
 			panic("session(release): " + err.Error())
 		}
@@ -252,12 +260,30 @@ func (m *Manager) sessionID() string {
 	return hex.EncodeToString(generateRandomKey(m.opt.IDLength / 2))
 }
 
+// validSessionID tests whether a provided session ID is a valid session ID.
+func (m *Manager) validSessionID(sid string) (bool, error) {
+	if len(sid) != m.opt.IDLength {
+		return false, errors.New("invalid 'sid': " + sid)
+	}
+
+	for i := range sid {
+		switch {
+		case '0' <= sid[i] && sid[i] <= '9':
+		case 'a' <= sid[i] && sid[i] <= 'f':
+		default:
+			return false, errors.New("invalid 'sid': " + sid)
+		}
+	}
+	return true, nil
+}
+
 // Start starts a session by generating new one
 // or retrieve existence one by reading session ID from HTTP request if it's valid.
 func (m *Manager) Start(ctx *macaron.Context) (RawStore, error) {
 	sid := ctx.GetCookie(m.opt.CookieName)
-	if len(sid) > 0 && m.provider.Exist(sid) {
-		return m.Read(sid)
+	valid, _ := m.validSessionID(sid)
+	if len(sid) > 0 && valid && m.provider.Exist(sid) {
+		return m.provider.Read(sid)
 	}
 
 	sid = m.sessionID()
@@ -284,10 +310,9 @@ func (m *Manager) Start(ctx *macaron.Context) (RawStore, error) {
 
 // Read returns raw session store by session ID.
 func (m *Manager) Read(sid string) (RawStore, error) {
-	// No slashes or dots "./" should ever occur in the sid and to prevent session file forgery bug.
-	// See https://github.com/gogs/gogs/issues/5469
-	if strings.ContainsAny(sid, "./") {
-		return nil, errors.New("invalid 'sid': " + sid)
+	// Ensure we're trying to read a valid session ID
+	if _, err := m.validSessionID(sid); err != nil {
+		return nil, err
 	}
 
 	return m.provider.Read(sid)
@@ -300,6 +325,10 @@ func (m *Manager) Destory(ctx *macaron.Context) error {
 		return nil
 	}
 
+	if _, err := m.validSessionID(sid); err != nil {
+		return err
+	}
+
 	if err := m.provider.Destory(sid); err != nil {
 		return err
 	}
@@ -318,11 +347,15 @@ func (m *Manager) Destory(ctx *macaron.Context) error {
 func (m *Manager) RegenerateId(ctx *macaron.Context) (sess RawStore, err error) {
 	sid := m.sessionID()
 	oldsid := ctx.GetCookie(m.opt.CookieName)
+	_, err = m.validSessionID(oldsid)
+	if err != nil {
+		return nil, err
+	}
 	sess, err = m.provider.Regenerate(oldsid, sid)
 	if err != nil {
 		return nil, err
 	}
-	ck := &http.Cookie{
+	cookie := &http.Cookie{
 		Name:     m.opt.CookieName,
 		Value:    sid,
 		Path:     m.opt.CookiePath,
@@ -331,10 +364,10 @@ func (m *Manager) RegenerateId(ctx *macaron.Context) (sess RawStore, err error)
 		Domain:   m.opt.Domain,
 	}
 	if m.opt.CookieLifeTime >= 0 {
-		ck.MaxAge = m.opt.CookieLifeTime
+		cookie.MaxAge = m.opt.CookieLifeTime
 	}
-	http.SetCookie(ctx.Resp, ck)
-	ctx.Req.AddCookie(ck)
+	http.SetCookie(ctx.Resp, cookie)
+	ctx.Req.AddCookie(cookie)
 	return sess, nil
 }
 
@@ -358,50 +391,3 @@ func (m *Manager) startGC() {
 func (m *Manager) SetSecure(secure bool) {
 	m.opt.Secure = secure
 }
-
-// ___________.____       _____    _________ ___ ___
-// \_   _____/|    |     /  _  \  /   _____//   |   \
-//  |    __)  |    |    /  /_\  \ \_____  \/    ~    \
-//  |     \   |    |___/    |    \/        \    Y    /
-//  \___  /   |_______ \____|__  /_______  /\___|_  /
-//      \/            \/       \/        \/       \/
-
-type Flash struct {
-	ctx *macaron.Context
-	url.Values
-	ErrorMsg, WarningMsg, InfoMsg, SuccessMsg string
-}
-
-func (f *Flash) set(name, msg string, current ...bool) {
-	isShow := false
-	if (len(current) == 0 && macaron.FlashNow) ||
-		(len(current) > 0 && current[0]) {
-		isShow = true
-	}
-
-	if isShow {
-		f.ctx.Data["Flash"] = f
-	} else {
-		f.Set(name, msg)
-	}
-}
-
-func (f *Flash) Error(msg string, current ...bool) {
-	f.ErrorMsg = msg
-	f.set("error", msg, current...)
-}
-
-func (f *Flash) Warning(msg string, current ...bool) {
-	f.WarningMsg = msg
-	f.set("warning", msg, current...)
-}
-
-func (f *Flash) Info(msg string, current ...bool) {
-	f.InfoMsg = msg
-	f.set("info", msg, current...)
-}
-
-func (f *Flash) Success(msg string, current ...bool) {
-	f.SuccessMsg = msg
-	f.set("success", msg, current...)
-}

vendor/github.com/go-macaron/session/utils.go

@@ -50,11 +50,14 @@ func DecodeGob(encoded []byte) (out map[interface{}]interface{}, err error) {
 	return out, err
 }
 
+// NOTE: A local copy in case of underlying package change
+var alphanum = []byte("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz")
+
 // generateRandomKey creates a random key with the given strength.
 func generateRandomKey(strength int) []byte {
 	k := make([]byte, strength)
 	if n, err := io.ReadFull(rand.Reader, k); n != strength || err != nil {
-		return com.RandomCreateBytes(strength)
+		return com.RandomCreateBytes(strength, alphanum...)
 	}
 	return k
 }

vendor/github.com/lunny/log/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2014 - 2016 lunny
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the {organization} nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/lunny/log/dbwriter.go

@@ -0,0 +1,36 @@
+package log
+
+import (
+	"database/sql"
+	"time"
+)
+
+type DBWriter struct {
+	db      *sql.DB
+	stmt    *sql.Stmt
+	content chan []byte
+}
+
+func NewDBWriter(db *sql.DB) (*DBWriter, error) {
+	_, err := db.Exec("CREATE TABLE IF NOT EXISTS log (id int, content text, created datetime)")
+	if err != nil {
+		return nil, err
+	}
+	stmt, err := db.Prepare("INSERT INTO log (content, created) values (?, ?)")
+	if err != nil {
+		return nil, err
+	}
+	return &DBWriter{db, stmt, make(chan []byte, 1000)}, nil
+}
+
+func (w *DBWriter) Write(p []byte) (n int, err error) {
+	_, err = w.stmt.Exec(string(p), time.Now())
+	if err == nil {
+		n = len(p)
+	}
+	return
+}
+
+func (w *DBWriter) Close() {
+	w.stmt.Close()
+}

vendor/github.com/lunny/log/filewriter.go

@@ -0,0 +1,112 @@
+package log
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+var _ io.Writer = &Files{}
+
+type ByType int
+
+const (
+	ByDay ByType = iota
+	ByHour
+	ByMonth
+)
+
+var (
+	formats = map[ByType]string{
+		ByDay:   "2006-01-02",
+		ByHour:  "2006-01-02-15",
+		ByMonth: "2006-01",
+	}
+)
+
+func SetFileFormat(t ByType, format string) {
+	formats[t] = format
+}
+
+func (b ByType) Format() string {
+	return formats[b]
+}
+
+type Files struct {
+	FileOptions
+	f          *os.File
+	lastFormat string
+	lock       sync.Mutex
+}
+
+type FileOptions struct {
+	Dir    string
+	ByType ByType
+	Loc    *time.Location
+}
+
+func prepareFileOption(opts []FileOptions) FileOptions {
+	var opt FileOptions
+	if len(opts) > 0 {
+		opt = opts[0]
+	}
+	if opt.Dir == "" {
+		opt.Dir = "./"
+	}
+	err := os.MkdirAll(opt.Dir, os.ModePerm)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	if opt.Loc == nil {
+		opt.Loc = time.Local
+	}
+	return opt
+}
+
+func NewFileWriter(opts ...FileOptions) *Files {
+	opt := prepareFileOption(opts)
+	return &Files{
+		FileOptions: opt,
+	}
+}
+
+func (f *Files) getFile() (*os.File, error) {
+	var err error
+	t := time.Now().In(f.Loc)
+	if f.f == nil {
+		f.lastFormat = t.Format(f.ByType.Format())
+		f.f, err = os.OpenFile(filepath.Join(f.Dir, f.lastFormat+".log"),
+			os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+		return f.f, err
+	}
+	if f.lastFormat != t.Format(f.ByType.Format()) {
+		f.f.Close()
+		f.lastFormat = t.Format(f.ByType.Format())
+		f.f, err = os.OpenFile(filepath.Join(f.Dir, f.lastFormat+".log"),
+			os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+		return f.f, err
+	}
+	return f.f, nil
+}
+
+func (f *Files) Write(bs []byte) (int, error) {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	w, err := f.getFile()
+	if err != nil {
+		return 0, err
+	}
+	return w.Write(bs)
+}
+
+func (f *Files) Close() {
+	if f.f != nil {
+		f.f.Close()
+		f.f = nil
+	}
+	f.lastFormat = ""
+}

vendor/github.com/lunny/log/logext.go

@@ -0,0 +1,595 @@
+package log
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+)
+
+// These flags define which text to prefix to each log entry generated by the Logger.
+const (
+	// Bits or'ed together to control what's printed. There is no control over the
+	// order they appear (the order listed here) or the format they present (as
+	// described in the comments).  A colon appears after these items:
+	//	2009/0123 01:23:23.123123 /a/b/c/d.go:23: message
+	Ldate         = 1 << iota     // the date: 2009/0123
+	Ltime                         // the time: 01:23:23
+	Lmicroseconds                 // microsecond resolution: 01:23:23.123123.  assumes Ltime.
+	Llongfile                     // full file name and line number: /a/b/c/d.go:23
+	Lshortfile                    // final file name element and line number: d.go:23. overrides Llongfile
+	Lmodule                       // module name
+	Llevel                        // level: 0(Debug), 1(Info), 2(Warn), 3(Error), 4(Panic), 5(Fatal)
+	Llongcolor                    // color will start [info] end of line
+	Lshortcolor                   // color only include [info]
+	LstdFlags     = Ldate | Ltime // initial values for the standard logger
+	//Ldefault      = Llevel | LstdFlags | Lshortfile | Llongcolor
+) // [prefix][time][level][module][shortfile|longfile]
+
+func Ldefault() int {
+	if runtime.GOOS == "windows" {
+		return Llevel | LstdFlags | Lshortfile
+	}
+	return Llevel | LstdFlags | Lshortfile | Llongcolor
+}
+
+func Version() string {
+	return "0.2.0.1121"
+}
+
+const (
+	Lall = iota
+)
+const (
+	Ldebug = iota
+	Linfo
+	Lwarn
+	Lerror
+	Lpanic
+	Lfatal
+	Lnone
+)
+
+const (
+	ForeBlack  = iota + 30 //30
+	ForeRed                //31
+	ForeGreen              //32
+	ForeYellow             //33
+	ForeBlue               //34
+	ForePurple             //35
+	ForeCyan               //36
+	ForeWhite              //37
+)
+
+const (
+	BackBlack  = iota + 40 //40
+	BackRed                //41
+	BackGreen              //42
+	BackYellow             //43
+	BackBlue               //44
+	BackPurple             //45
+	BackCyan               //46
+	BackWhite              //47
+)
+
+var levels = []string{
+	"[Debug]",
+	"[Info]",
+	"[Warn]",
+	"[Error]",
+	"[Panic]",
+	"[Fatal]",
+}
+
+// MUST called before all logs
+func SetLevels(lvs []string) {
+	levels = lvs
+}
+
+var colors = []int{
+	ForeCyan,
+	ForeGreen,
+	ForeYellow,
+	ForeRed,
+	ForePurple,
+	ForeBlue,
+}
+
+// MUST called before all logs
+func SetColors(cls []int) {
+	colors = cls
+}
+
+// A Logger represents an active logging object that generates lines of
+// output to an io.Writer.  Each logging operation makes a single call to
+// the Writer's Write method.  A Logger can be used simultaneously from
+// multiple goroutines; it guarantees to serialize access to the Writer.
+type Logger struct {
+	mu         sync.Mutex // ensures atomic writes; protects the following fields
+	prefix     string     // prefix to write at beginning of each line
+	flag       int        // properties
+	Level      int
+	out        io.Writer    // destination for output
+	buf        bytes.Buffer // for accumulating text to write
+	levelStats [6]int64
+	loc        *time.Location
+}
+
+// New creates a new Logger.   The out variable sets the
+// destination to which log data will be written.
+// The prefix appears at the beginning of each generated log line.
+// The flag argument defines the logging properties.
+func New(out io.Writer, prefix string, flag int) *Logger {
+	l := &Logger{out: out, prefix: prefix, Level: 1, flag: flag, loc: time.Local}
+	if out != os.Stdout {
+		l.flag = RmColorFlags(l.flag)
+	}
+	return l
+}
+
+var Std = New(os.Stderr, "", Ldefault())
+
+// Cheap integer to fixed-width decimal ASCII.  Give a negative width to avoid zero-padding.
+// Knows the buffer has capacity.
+func itoa(buf *bytes.Buffer, i int, wid int) {
+	var u uint = uint(i)
+	if u == 0 && wid <= 1 {
+		buf.WriteByte('0')
+		return
+	}
+
+	// Assemble decimal in reverse order.
+	var b [32]byte
+	bp := len(b)
+	for ; u > 0 || wid > 0; u /= 10 {
+		bp--
+		wid--
+		b[bp] = byte(u%10) + '0'
+	}
+
+	// avoid slicing b to avoid an allocation.
+	for bp < len(b) {
+		buf.WriteByte(b[bp])
+		bp++
+	}
+}
+
+func moduleOf(file string) string {
+	pos := strings.LastIndex(file, "/")
+	if pos != -1 {
+		pos1 := strings.LastIndex(file[:pos], "/src/")
+		if pos1 != -1 {
+			return file[pos1+5 : pos]
+		}
+	}
+	return "UNKNOWN"
+}
+
+func (l *Logger) formatHeader(buf *bytes.Buffer, t time.Time,
+	file string, line int, lvl int, reqId string) {
+	if l.prefix != "" {
+		buf.WriteString(l.prefix)
+	}
+	if l.flag&(Ldate|Ltime|Lmicroseconds) != 0 {
+		if l.flag&Ldate != 0 {
+			year, month, day := t.Date()
+			itoa(buf, year, 4)
+			buf.WriteByte('/')
+			itoa(buf, int(month), 2)
+			buf.WriteByte('/')
+			itoa(buf, day, 2)
+			buf.WriteByte(' ')
+		}
+		if l.flag&(Ltime|Lmicroseconds) != 0 {
+			hour, min, sec := t.Clock()
+			itoa(buf, hour, 2)
+			buf.WriteByte(':')
+			itoa(buf, min, 2)
+			buf.WriteByte(':')
+			itoa(buf, sec, 2)
+			if l.flag&Lmicroseconds != 0 {
+				buf.WriteByte('.')
+				itoa(buf, t.Nanosecond()/1e3, 6)
+			}
+			buf.WriteByte(' ')
+		}
+	}
+	if reqId != "" {
+		buf.WriteByte('[')
+		buf.WriteString(reqId)
+		buf.WriteByte(']')
+		buf.WriteByte(' ')
+	}
+
+	if l.flag&(Lshortcolor|Llongcolor) != 0 {
+		buf.WriteString(fmt.Sprintf("\033[1;%dm", colors[lvl]))
+	}
+	if l.flag&Llevel != 0 {
+		buf.WriteString(levels[lvl])
+		buf.WriteByte(' ')
+	}
+	if l.flag&Lshortcolor != 0 {
+		buf.WriteString("\033[0m")
+	}
+
+	if l.flag&Lmodule != 0 {
+		buf.WriteByte('[')
+		buf.WriteString(moduleOf(file))
+		buf.WriteByte(']')
+		buf.WriteByte(' ')
+	}
+	if l.flag&(Lshortfile|Llongfile) != 0 {
+		if l.flag&Lshortfile != 0 {
+			short := file
+			for i := len(file) - 1; i > 0; i-- {
+				if file[i] == '/' {
+					short = file[i+1:]
+					break
+				}
+			}
+			file = short
+		}
+		buf.WriteString(file)
+		buf.WriteByte(':')
+		itoa(buf, line, -1)
+		buf.WriteByte(' ')
+	}
+}
+
+// Output writes the output for a logging event.  The string s contains
+// the text to print after the prefix specified by the flags of the
+// Logger.  A newline is appended if the last character of s is not
+// already a newline.  Calldepth is used to recover the PC and is
+// provided for generality, although at the moment on all pre-defined
+// paths it will be 2.
+func (l *Logger) Output(reqId string, lvl int, calldepth int, s string) error {
+	if lvl < l.Level {
+		return nil
+	}
+	now := time.Now().In(l.loc) // get this early.
+	var file string
+	var line int
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.flag&(Lshortfile|Llongfile|Lmodule) != 0 {
+		// release lock while getting caller info - it's expensive.
+		l.mu.Unlock()
+		var ok bool
+		_, file, line, ok = runtime.Caller(calldepth)
+		if !ok {
+			file = "???"
+			line = 0
+		}
+		l.mu.Lock()
+	}
+	l.levelStats[lvl]++
+	l.buf.Reset()
+	l.formatHeader(&l.buf, now, file, line, lvl, reqId)
+	l.buf.WriteString(s)
+	if l.flag&Llongcolor != 0 {
+		l.buf.WriteString("\033[0m")
+	}
+	if len(s) > 0 && s[len(s)-1] != '\n' {
+		l.buf.WriteByte('\n')
+	}
+	_, err := l.out.Write(l.buf.Bytes())
+	return err
+}
+
+// -----------------------------------------
+
+// Printf calls l.Output to print to the logger.
+// Arguments are handled in the manner of fmt.Printf.
+func (l *Logger) Printf(format string, v ...interface{}) {
+	l.Output("", Linfo, 2, fmt.Sprintf(format, v...))
+}
+
+// Print calls l.Output to print to the logger.
+// Arguments are handled in the manner of fmt.Print.
+func (l *Logger) Print(v ...interface{}) {
+	l.Output("", Linfo, 2, fmt.Sprint(v...))
+}
+
+// Println calls l.Output to print to the logger.
+// Arguments are handled in the manner of fmt.Println.
+func (l *Logger) Println(v ...interface{}) {
+	l.Output("", Linfo, 2, fmt.Sprintln(v...))
+}
+
+// -----------------------------------------
+
+func (l *Logger) Debugf(format string, v ...interface{}) {
+	l.Output("", Ldebug, 2, fmt.Sprintf(format, v...))
+}
+
+func (l *Logger) Debug(v ...interface{}) {
+	l.Output("", Ldebug, 2, fmt.Sprintln(v...))
+}
+
+// -----------------------------------------
+func (l *Logger) Infof(format string, v ...interface{}) {
+	l.Output("", Linfo, 2, fmt.Sprintf(format, v...))
+}
+
+func (l *Logger) Info(v ...interface{}) {
+	l.Output("", Linfo, 2, fmt.Sprintln(v...))
+}
+
+// -----------------------------------------
+func (l *Logger) Warnf(format string, v ...interface{}) {
+	l.Output("", Lwarn, 2, fmt.Sprintf(format, v...))
+}
+
+func (l *Logger) Warn(v ...interface{}) {
+	l.Output("", Lwarn, 2, fmt.Sprintln(v...))
+}
+
+// -----------------------------------------
+
+func (l *Logger) Errorf(format string, v ...interface{}) {
+	l.Output("", Lerror, 2, fmt.Sprintf(format, v...))
+}
+
+func (l *Logger) Error(v ...interface{}) {
+	l.Output("", Lerror, 2, fmt.Sprintln(v...))
+}
+
+// -----------------------------------------
+
+func (l *Logger) Fatal(v ...interface{}) {
+	l.Output("", Lfatal, 2, fmt.Sprintln(v...))
+	os.Exit(1)
+}
+
+// Fatalf is equivalent to l.Printf() followed by a call to os.Exit(1).
+func (l *Logger) Fatalf(format string, v ...interface{}) {
+	l.Output("", Lfatal, 2, fmt.Sprintf(format, v...))
+	os.Exit(1)
+}
+
+// -----------------------------------------
+// Panic is equivalent to l.Print() followed by a call to panic().
+func (l *Logger) Panic(v ...interface{}) {
+	s := fmt.Sprintln(v...)
+	l.Output("", Lpanic, 2, s)
+	panic(s)
+}
+
+// Panicf is equivalent to l.Printf() followed by a call to panic().
+func (l *Logger) Panicf(format string, v ...interface{}) {
+	s := fmt.Sprintf(format, v...)
+	l.Output("", Lpanic, 2, s)
+	panic(s)
+}
+
+// -----------------------------------------
+func (l *Logger) Stack(v ...interface{}) {
+	s := fmt.Sprint(v...)
+	s += "\n"
+	buf := make([]byte, 1024*1024)
+	n := runtime.Stack(buf, true)
+	s += string(buf[:n])
+	s += "\n"
+	l.Output("", Lerror, 2, s)
+}
+
+// -----------------------------------------
+func (l *Logger) Stat() (stats []int64) {
+	l.mu.Lock()
+	v := l.levelStats
+	l.mu.Unlock()
+	return v[:]
+}
+
+// Flags returns the output flags for the logger.
+func (l *Logger) Flags() int {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.flag
+}
+
+func RmColorFlags(flag int) int {
+	// for un std out, it should not show color since almost them don't support
+	if flag&Llongcolor != 0 {
+		flag = flag ^ Llongcolor
+	}
+	if flag&Lshortcolor != 0 {
+		flag = flag ^ Lshortcolor
+	}
+	return flag
+}
+
+func (l *Logger) Location() *time.Location {
+	return l.loc
+}
+
+func (l *Logger) SetLocation(loc *time.Location) {
+	l.loc = loc
+}
+
+// SetFlags sets the output flags for the logger.
+func (l *Logger) SetFlags(flag int) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.out != os.Stdout {
+		flag = RmColorFlags(flag)
+	}
+	l.flag = flag
+}
+
+// Prefix returns the output prefix for the logger.
+func (l *Logger) Prefix() string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.prefix
+}
+
+// SetPrefix sets the output prefix for the logger.
+func (l *Logger) SetPrefix(prefix string) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.prefix = prefix
+}
+
+// SetOutputLevel sets the output level for the logger.
+func (l *Logger) SetOutputLevel(lvl int) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.Level = lvl
+}
+
+func (l *Logger) OutputLevel() int {
+	return l.Level
+}
+
+func (l *Logger) SetOutput(w io.Writer) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.out = w
+	if w != os.Stdout {
+		l.flag = RmColorFlags(l.flag)
+	}
+}
+
+// SetOutput sets the output destination for the standard logger.
+func SetOutput(w io.Writer) {
+	Std.SetOutput(w)
+}
+
+func SetLocation(loc *time.Location) {
+	Std.SetLocation(loc)
+}
+
+func Location() *time.Location {
+	return Std.Location()
+}
+
+// Flags returns the output flags for the standard logger.
+func Flags() int {
+	return Std.Flags()
+}
+
+// SetFlags sets the output flags for the standard logger.
+func SetFlags(flag int) {
+	Std.SetFlags(flag)
+}
+
+// Prefix returns the output prefix for the standard logger.
+func Prefix() string {
+	return Std.Prefix()
+}
+
+// SetPrefix sets the output prefix for the standard logger.
+func SetPrefix(prefix string) {
+	Std.SetPrefix(prefix)
+}
+
+func SetOutputLevel(lvl int) {
+	Std.SetOutputLevel(lvl)
+}
+
+func OutputLevel() int {
+	return Std.OutputLevel()
+}
+
+// -----------------------------------------
+
+// Print calls Output to print to the standard logger.
+// Arguments are handled in the manner of fmt.Print.
+func Print(v ...interface{}) {
+	Std.Output("", Linfo, 2, fmt.Sprintln(v...))
+}
+
+// Printf calls Output to print to the standard logger.
+// Arguments are handled in the manner of fmt.Printf.
+func Printf(format string, v ...interface{}) {
+	Std.Output("", Linfo, 2, fmt.Sprintf(format, v...))
+}
+
+// Println calls Output to print to the standard logger.
+// Arguments are handled in the manner of fmt.Println.
+func Println(v ...interface{}) {
+	Std.Output("", Linfo, 2, fmt.Sprintln(v...))
+}
+
+// -----------------------------------------
+
+func Debugf(format string, v ...interface{}) {
+	Std.Output("", Ldebug, 2, fmt.Sprintf(format, v...))
+}
+
+func Debug(v ...interface{}) {
+	Std.Output("", Ldebug, 2, fmt.Sprintln(v...))
+}
+
+// -----------------------------------------
+
+func Infof(format string, v ...interface{}) {
+	Std.Output("", Linfo, 2, fmt.Sprintf(format, v...))
+}
+
+func Info(v ...interface{}) {
+	Std.Output("", Linfo, 2, fmt.Sprintln(v...))
+}
+
+// -----------------------------------------
+
+func Warnf(format string, v ...interface{}) {
+	Std.Output("", Lwarn, 2, fmt.Sprintf(format, v...))
+}
+
+func Warn(v ...interface{}) {
+	Std.Output("", Lwarn, 2, fmt.Sprintln(v...))
+}
+
+// -----------------------------------------
+
+func Errorf(format string, v ...interface{}) {
+	Std.Output("", Lerror, 2, fmt.Sprintf(format, v...))
+}
+
+func Error(v ...interface{}) {
+	Std.Output("", Lerror, 2, fmt.Sprintln(v...))
+}
+
+// -----------------------------------------
+
+// Fatal is equivalent to Print() followed by a call to os.Exit(1).
+func Fatal(v ...interface{}) {
+	Std.Output("", Lfatal, 2, fmt.Sprintln(v...))
+}
+
+// Fatalf is equivalent to Printf() followed by a call to os.Exit(1).
+func Fatalf(format string, v ...interface{}) {
+	Std.Output("", Lfatal, 2, fmt.Sprintf(format, v...))
+}
+
+// -----------------------------------------
+
+// Panic is equivalent to Print() followed by a call to panic().
+func Panic(v ...interface{}) {
+	Std.Output("", Lpanic, 2, fmt.Sprintln(v...))
+}
+
+// Panicf is equivalent to Printf() followed by a call to panic().
+func Panicf(format string, v ...interface{}) {
+	Std.Output("", Lpanic, 2, fmt.Sprintf(format, v...))
+}
+
+// -----------------------------------------
+
+func Stack(v ...interface{}) {
+	s := fmt.Sprint(v...)
+	s += "\n"
+	buf := make([]byte, 1024*1024)
+	n := runtime.Stack(buf, true)
+	s += string(buf[:n])
+	s += "\n"
+	Std.Output("", Lerror, 2, s)
+}
+
+// -----------------------------------------

vendor/github.com/lunny/nodb/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 siddontang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/lunny/nodb/batch.go

@@ -0,0 +1,106 @@
+package nodb
+
+import (
+	"sync"
+
+	"github.com/lunny/nodb/store"
+)
+
+type batch struct {
+	l *Nodb
+
+	store.WriteBatch
+
+	sync.Locker
+
+	logs [][]byte
+
+	tx *Tx
+}
+
+func (b *batch) Commit() error {
+	b.l.commitLock.Lock()
+	defer b.l.commitLock.Unlock()
+
+	err := b.WriteBatch.Commit()
+
+	if b.l.binlog != nil {
+		if err == nil {
+			if b.tx == nil {
+				b.l.binlog.Log(b.logs...)
+			} else {
+				b.tx.logs = append(b.tx.logs, b.logs...)
+			}
+		}
+		b.logs = [][]byte{}
+	}
+
+	return err
+}
+
+func (b *batch) Lock() {
+	b.Locker.Lock()
+}
+
+func (b *batch) Unlock() {
+	if b.l.binlog != nil {
+		b.logs = [][]byte{}
+	}
+	b.WriteBatch.Rollback()
+	b.Locker.Unlock()
+}
+
+func (b *batch) Put(key []byte, value []byte) {
+	if b.l.binlog != nil {
+		buf := encodeBinLogPut(key, value)
+		b.logs = append(b.logs, buf)
+	}
+	b.WriteBatch.Put(key, value)
+}
+
+func (b *batch) Delete(key []byte) {
+	if b.l.binlog != nil {
+		buf := encodeBinLogDelete(key)
+		b.logs = append(b.logs, buf)
+	}
+	b.WriteBatch.Delete(key)
+}
+
+type dbBatchLocker struct {
+	l      *sync.Mutex
+	wrLock *sync.RWMutex
+}
+
+func (l *dbBatchLocker) Lock() {
+	l.wrLock.RLock()
+	l.l.Lock()
+}
+
+func (l *dbBatchLocker) Unlock() {
+	l.l.Unlock()
+	l.wrLock.RUnlock()
+}
+
+type txBatchLocker struct {
+}
+
+func (l *txBatchLocker) Lock()   {}
+func (l *txBatchLocker) Unlock() {}
+
+type multiBatchLocker struct {
+}
+
+func (l *multiBatchLocker) Lock()   {}
+func (l *multiBatchLocker) Unlock() {}
+
+func (l *Nodb) newBatch(wb store.WriteBatch, locker sync.Locker, tx *Tx) *batch {
+	b := new(batch)
+	b.l = l
+	b.WriteBatch = wb
+
+	b.tx = tx
+	b.Locker = locker
+
+	b.logs = [][]byte{}
+	return b
+}

vendor/github.com/lunny/nodb/binlog.go

@@ -0,0 +1,391 @@
+package nodb
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/lunny/log"
+	"github.com/lunny/nodb/config"
+)
+
+type BinLogHead struct {
+	CreateTime uint32
+	BatchId    uint32
+	PayloadLen uint32
+}
+
+func (h *BinLogHead) Len() int {
+	return 12
+}
+
+func (h *BinLogHead) Write(w io.Writer) error {
+	if err := binary.Write(w, binary.BigEndian, h.CreateTime); err != nil {
+		return err
+	}
+
+	if err := binary.Write(w, binary.BigEndian, h.BatchId); err != nil {
+		return err
+	}
+
+	if err := binary.Write(w, binary.BigEndian, h.PayloadLen); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (h *BinLogHead) handleReadError(err error) error {
+	if err == io.EOF {
+		return io.ErrUnexpectedEOF
+	} else {
+		return err
+	}
+}
+
+func (h *BinLogHead) Read(r io.Reader) error {
+	var err error
+	if err = binary.Read(r, binary.BigEndian, &h.CreateTime); err != nil {
+		return err
+	}
+
+	if err = binary.Read(r, binary.BigEndian, &h.BatchId); err != nil {
+		return h.handleReadError(err)
+	}
+
+	if err = binary.Read(r, binary.BigEndian, &h.PayloadLen); err != nil {
+		return h.handleReadError(err)
+	}
+
+	return nil
+}
+
+func (h *BinLogHead) InSameBatch(ho *BinLogHead) bool {
+	if h.CreateTime == ho.CreateTime && h.BatchId == ho.BatchId {
+		return true
+	} else {
+		return false
+	}
+}
+
+/*
+index file format:
+ledis-bin.00001
+ledis-bin.00002
+ledis-bin.00003
+
+log file format
+
+Log: Head|PayloadData
+
+Head: createTime|batchId|payloadData
+
+*/
+
+type BinLog struct {
+	sync.Mutex
+
+	path string
+
+	cfg *config.BinLogConfig
+
+	logFile *os.File
+
+	logWb *bufio.Writer
+
+	indexName    string
+	logNames     []string
+	lastLogIndex int64
+
+	batchId uint32
+
+	ch chan struct{}
+}
+
+func NewBinLog(cfg *config.Config) (*BinLog, error) {
+	l := new(BinLog)
+
+	l.cfg = &cfg.BinLog
+	l.cfg.Adjust()
+
+	l.path = path.Join(cfg.DataDir, "binlog")
+
+	if err := os.MkdirAll(l.path, os.ModePerm); err != nil {
+		return nil, err
+	}
+
+	l.logNames = make([]string, 0, 16)
+
+	l.ch = make(chan struct{})
+
+	if err := l.loadIndex(); err != nil {
+		return nil, err
+	}
+
+	return l, nil
+}
+
+func (l *BinLog) flushIndex() error {
+	data := strings.Join(l.logNames, "\n")
+
+	bakName := fmt.Sprintf("%s.bak", l.indexName)
+	f, err := os.OpenFile(bakName, os.O_WRONLY|os.O_CREATE, 0666)
+	if err != nil {
+		log.Error("create binlog bak index error %s", err.Error())
+		return err
+	}
+
+	if _, err := f.WriteString(data); err != nil {
+		log.Error("write binlog index error %s", err.Error())
+		f.Close()
+		return err
+	}
+
+	f.Close()
+
+	if err := os.Rename(bakName, l.indexName); err != nil {
+		log.Error("rename binlog bak index error %s", err.Error())
+		return err
+	}
+
+	return nil
+}
+
+func (l *BinLog) loadIndex() error {
+	l.indexName = path.Join(l.path, fmt.Sprintf("ledis-bin.index"))
+	if _, err := os.Stat(l.indexName); os.IsNotExist(err) {
+		//no index file, nothing to do
+	} else {
+		indexData, err := ioutil.ReadFile(l.indexName)
+		if err != nil {
+			return err
+		}
+
+		lines := strings.Split(string(indexData), "\n")
+		for _, line := range lines {
+			line = strings.Trim(line, "\r\n ")
+			if len(line) == 0 {
+				continue
+			}
+
+			if _, err := os.Stat(path.Join(l.path, line)); err != nil {
+				log.Error("load index line %s error %s", line, err.Error())
+				return err
+			} else {
+				l.logNames = append(l.logNames, line)
+			}
+		}
+	}
+	if l.cfg.MaxFileNum > 0 && len(l.logNames) > l.cfg.MaxFileNum {
+		//remove oldest logfile
+		if err := l.Purge(len(l.logNames) - l.cfg.MaxFileNum); err != nil {
+			return err
+		}
+	}
+
+	var err error
+	if len(l.logNames) == 0 {
+		l.lastLogIndex = 1
+	} else {
+		lastName := l.logNames[len(l.logNames)-1]
+
+		if l.lastLogIndex, err = strconv.ParseInt(path.Ext(lastName)[1:], 10, 64); err != nil {
+			log.Error("invalid logfile name %s", err.Error())
+			return err
+		}
+
+		//like mysql, if server restart, a new binlog will create
+		l.lastLogIndex++
+	}
+
+	return nil
+}
+
+func (l *BinLog) getLogFile() string {
+	return l.FormatLogFileName(l.lastLogIndex)
+}
+
+func (l *BinLog) openNewLogFile() error {
+	var err error
+	lastName := l.getLogFile()
+
+	logPath := path.Join(l.path, lastName)
+	if l.logFile, err = os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY, 0666); err != nil {
+		log.Error("open new logfile error %s", err.Error())
+		return err
+	}
+
+	if l.cfg.MaxFileNum > 0 && len(l.logNames) == l.cfg.MaxFileNum {
+		l.purge(1)
+	}
+
+	l.logNames = append(l.logNames, lastName)
+
+	if l.logWb == nil {
+		l.logWb = bufio.NewWriterSize(l.logFile, 1024)
+	} else {
+		l.logWb.Reset(l.logFile)
+	}
+
+	if err = l.flushIndex(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (l *BinLog) checkLogFileSize() bool {
+	if l.logFile == nil {
+		return false
+	}
+
+	st, _ := l.logFile.Stat()
+	if st.Size() >= int64(l.cfg.MaxFileSize) {
+		l.closeLog()
+		return true
+	}
+
+	return false
+}
+
+func (l *BinLog) closeLog() {
+	l.lastLogIndex++
+
+	l.logFile.Close()
+	l.logFile = nil
+}
+
+func (l *BinLog) purge(n int) {
+	for i := 0; i < n; i++ {
+		logPath := path.Join(l.path, l.logNames[i])
+		os.Remove(logPath)
+	}
+
+	copy(l.logNames[0:], l.logNames[n:])
+	l.logNames = l.logNames[0 : len(l.logNames)-n]
+}
+
+func (l *BinLog) Close() {
+	if l.logFile != nil {
+		l.logFile.Close()
+		l.logFile = nil
+	}
+}
+
+func (l *BinLog) LogNames() []string {
+	return l.logNames
+}
+
+func (l *BinLog) LogFileName() string {
+	return l.getLogFile()
+}
+
+func (l *BinLog) LogFilePos() int64 {
+	if l.logFile == nil {
+		return 0
+	} else {
+		st, _ := l.logFile.Stat()
+		return st.Size()
+	}
+}
+
+func (l *BinLog) LogFileIndex() int64 {
+	return l.lastLogIndex
+}
+
+func (l *BinLog) FormatLogFileName(index int64) string {
+	return fmt.Sprintf("ledis-bin.%07d", index)
+}
+
+func (l *BinLog) FormatLogFilePath(index int64) string {
+	return path.Join(l.path, l.FormatLogFileName(index))
+}
+
+func (l *BinLog) LogPath() string {
+	return l.path
+}
+
+func (l *BinLog) Purge(n int) error {
+	l.Lock()
+	defer l.Unlock()
+
+	if len(l.logNames) == 0 {
+		return nil
+	}
+
+	if n >= len(l.logNames) {
+		n = len(l.logNames)
+		//can not purge current log file
+		if l.logNames[n-1] == l.getLogFile() {
+			n = n - 1
+		}
+	}
+
+	l.purge(n)
+
+	return l.flushIndex()
+}
+
+func (l *BinLog) PurgeAll() error {
+	l.Lock()
+	defer l.Unlock()
+
+	l.closeLog()
+	return l.openNewLogFile()
+}
+
+func (l *BinLog) Log(args ...[]byte) error {
+	l.Lock()
+	defer l.Unlock()
+
+	var err error
+
+	if l.logFile == nil {
+		if err = l.openNewLogFile(); err != nil {
+			return err
+		}
+	}
+
+	head := &BinLogHead{}
+
+	head.CreateTime = uint32(time.Now().Unix())
+	head.BatchId = l.batchId
+
+	l.batchId++
+
+	for _, data := range args {
+		head.PayloadLen = uint32(len(data))
+
+		if err := head.Write(l.logWb); err != nil {
+			return err
+		}
+
+		if _, err := l.logWb.Write(data); err != nil {
+			return err
+		}
+	}
+
+	if err = l.logWb.Flush(); err != nil {
+		log.Error("write log error %s", err.Error())
+		return err
+	}
+
+	l.checkLogFileSize()
+
+	close(l.ch)
+	l.ch = make(chan struct{})
+
+	return nil
+}
+
+func (l *BinLog) Wait() <-chan struct{} {
+	return l.ch
+}

vendor/github.com/lunny/nodb/binlog_util.go

@@ -0,0 +1,215 @@
+package nodb
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"strconv"
+)
+
+var (
+	errBinLogDeleteType  = errors.New("invalid bin log delete type")
+	errBinLogPutType     = errors.New("invalid bin log put type")
+	errBinLogCommandType = errors.New("invalid bin log command type")
+)
+
+func encodeBinLogDelete(key []byte) []byte {
+	buf := make([]byte, 1+len(key))
+	buf[0] = BinLogTypeDeletion
+	copy(buf[1:], key)
+	return buf
+}
+
+func decodeBinLogDelete(sz []byte) ([]byte, error) {
+	if len(sz) < 1 || sz[0] != BinLogTypeDeletion {
+		return nil, errBinLogDeleteType
+	}
+
+	return sz[1:], nil
+}
+
+func encodeBinLogPut(key []byte, value []byte) []byte {
+	buf := make([]byte, 3+len(key)+len(value))
+	buf[0] = BinLogTypePut
+	pos := 1
+	binary.BigEndian.PutUint16(buf[pos:], uint16(len(key)))
+	pos += 2
+	copy(buf[pos:], key)
+	pos += len(key)
+	copy(buf[pos:], value)
+
+	return buf
+}
+
+func decodeBinLogPut(sz []byte) ([]byte, []byte, error) {
+	if len(sz) < 3 || sz[0] != BinLogTypePut {
+		return nil, nil, errBinLogPutType
+	}
+
+	keyLen := int(binary.BigEndian.Uint16(sz[1:]))
+	if 3+keyLen > len(sz) {
+		return nil, nil, errBinLogPutType
+	}
+
+	return sz[3 : 3+keyLen], sz[3+keyLen:], nil
+}
+
+func FormatBinLogEvent(event []byte) (string, error) {
+	logType := uint8(event[0])
+
+	var err error
+	var k []byte
+	var v []byte
+
+	var buf []byte = make([]byte, 0, 1024)
+
+	switch logType {
+	case BinLogTypePut:
+		k, v, err = decodeBinLogPut(event)
+		buf = append(buf, "PUT "...)
+	case BinLogTypeDeletion:
+		k, err = decodeBinLogDelete(event)
+		buf = append(buf, "DELETE "...)
+	default:
+		err = errInvalidBinLogEvent
+	}
+
+	if err != nil {
+		return "", err
+	}
+
+	if buf, err = formatDataKey(buf, k); err != nil {
+		return "", err
+	}
+
+	if v != nil && len(v) != 0 {
+		buf = append(buf, fmt.Sprintf(" %q", v)...)
+	}
+
+	return String(buf), nil
+}
+
+func formatDataKey(buf []byte, k []byte) ([]byte, error) {
+	if len(k) < 2 {
+		return nil, errInvalidBinLogEvent
+	}
+
+	buf = append(buf, fmt.Sprintf("DB:%2d ", k[0])...)
+	buf = append(buf, fmt.Sprintf("%s ", TypeName[k[1]])...)
+
+	db := new(DB)
+	db.index = k[0]
+
+	//to do format at respective place
+
+	switch k[1] {
+	case KVType:
+		if key, err := db.decodeKVKey(k); err != nil {
+			return nil, err
+		} else {
+			buf = strconv.AppendQuote(buf, String(key))
+		}
+	case HashType:
+		if key, field, err := db.hDecodeHashKey(k); err != nil {
+			return nil, err
+		} else {
+			buf = strconv.AppendQuote(buf, String(key))
+			buf = append(buf, ' ')
+			buf = strconv.AppendQuote(buf, String(field))
+		}
+	case HSizeType:
+		if key, err := db.hDecodeSizeKey(k); err != nil {
+			return nil, err
+		} else {
+			buf = strconv.AppendQuote(buf, String(key))
+		}
+	case ListType:
+		if key, seq, err := db.lDecodeListKey(k); err != nil {
+			return nil, err
+		} else {
+			buf = strconv.AppendQuote(buf, String(key))
+			buf = append(buf, ' ')
+			buf = strconv.AppendInt(buf, int64(seq), 10)
+		}
+	case LMetaType:
+		if key, err := db.lDecodeMetaKey(k); err != nil {
+			return nil, err
+		} else {
+			buf = strconv.AppendQuote(buf, String(key))
+		}
+	case ZSetType:
+		if key, m, err := db.zDecodeSetKey(k); err != nil {
+			return nil, err
+		} else {
+			buf = strconv.AppendQuote(buf, String(key))
+			buf = append(buf, ' ')
+			buf = strconv.AppendQuote(buf, String(m))
+		}
+	case ZSizeType:
+		if key, err := db.zDecodeSizeKey(k); err != nil {
+			return nil, err
+		} else {
+			buf = strconv.AppendQuote(buf, String(key))
+		}
+	case ZScoreType:
+		if key, m, score, err := db.zDecodeScoreKey(k); err != nil {
+			return nil, err
+		} else {
+			buf = strconv.AppendQuote(buf, String(key))
+			buf = append(buf, ' ')
+			buf = strconv.AppendQuote(buf, String(m))
+			buf = append(buf, ' ')
+			buf = strconv.AppendInt(buf, score, 10)
+		}
+	case BitType:
+		if key, seq, err := db.bDecodeBinKey(k); err != nil {
+			return nil, err
+		} else {
+			buf = strconv.AppendQuote(buf, String(key))
+			buf = append(buf, ' ')
+			buf = strconv.AppendUint(buf, uint64(seq), 10)
+		}
+	case BitMetaType:
+		if key, err := db.bDecodeMetaKey(k); err != nil {
+			return nil, err
+		} else {
+			buf = strconv.AppendQuote(buf, String(key))
+		}
+	case SetType:
+		if key, member, err := db.sDecodeSetKey(k); err != nil {
+			return nil, err
+		} else {
+			buf = strconv.AppendQuote(buf, String(key))
+			buf = append(buf, ' ')
+			buf = strconv.AppendQuote(buf, String(member))
+		}
+	case SSizeType:
+		if key, err := db.sDecodeSizeKey(k); err != nil {
+			return nil, err
+		} else {
+			buf = strconv.AppendQuote(buf, String(key))
+		}
+	case ExpTimeType:
+		if tp, key, t, err := db.expDecodeTimeKey(k); err != nil {
+			return nil, err
+		} else {
+			buf = append(buf, TypeName[tp]...)
+			buf = append(buf, ' ')
+			buf = strconv.AppendQuote(buf, String(key))
+			buf = append(buf, ' ')
+			buf = strconv.AppendInt(buf, t, 10)
+		}
+	case ExpMetaType:
+		if tp, key, err := db.expDecodeMetaKey(k); err != nil {
+			return nil, err
+		} else {
+			buf = append(buf, TypeName[tp]...)
+			buf = append(buf, ' ')
+			buf = strconv.AppendQuote(buf, String(key))
+		}
+	default:
+		return nil, errInvalidBinLogEvent
+	}
+
+	return buf, nil
+}

vendor/github.com/lunny/nodb/config/config.go

@@ -0,0 +1,135 @@
+package config
+
+import (
+	"io/ioutil"
+
+	"github.com/BurntSushi/toml"
+)
+
+type Size int
+
+const (
+	DefaultAddr     string = "127.0.0.1:6380"
+	DefaultHttpAddr string = "127.0.0.1:11181"
+
+	DefaultDBName string = "goleveldb"
+
+	DefaultDataDir string = "./data"
+)
+
+const (
+	MaxBinLogFileSize int = 1024 * 1024 * 1024
+	MaxBinLogFileNum  int = 10000
+
+	DefaultBinLogFileSize int = MaxBinLogFileSize
+	DefaultBinLogFileNum  int = 10
+)
+
+type LevelDBConfig struct {
+	Compression     bool `toml:"compression"`
+	BlockSize       int  `toml:"block_size"`
+	WriteBufferSize int  `toml:"write_buffer_size"`
+	CacheSize       int  `toml:"cache_size"`
+	MaxOpenFiles    int  `toml:"max_open_files"`
+}
+
+type LMDBConfig struct {
+	MapSize int  `toml:"map_size"`
+	NoSync  bool `toml:"nosync"`
+}
+
+type BinLogConfig struct {
+	MaxFileSize int `toml:"max_file_size"`
+	MaxFileNum  int `toml:"max_file_num"`
+}
+
+type Config struct {
+	DataDir string `toml:"data_dir"`
+
+	DBName string `toml:"db_name"`
+
+	LevelDB LevelDBConfig `toml:"leveldb"`
+
+	LMDB LMDBConfig `toml:"lmdb"`
+
+	BinLog BinLogConfig `toml:"binlog"`
+
+	SlaveOf string `toml:"slaveof"`
+
+	AccessLog string `toml:"access_log"`
+}
+
+func NewConfigWithFile(fileName string) (*Config, error) {
+	data, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewConfigWithData(data)
+}
+
+func NewConfigWithData(data []byte) (*Config, error) {
+	cfg := NewConfigDefault()
+
+	_, err := toml.Decode(string(data), cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return cfg, nil
+}
+
+func NewConfigDefault() *Config {
+	cfg := new(Config)
+
+	cfg.DataDir = DefaultDataDir
+
+	cfg.DBName = DefaultDBName
+
+	// disable binlog
+	cfg.BinLog.MaxFileNum = 0
+	cfg.BinLog.MaxFileSize = 0
+
+	// disable replication
+	cfg.SlaveOf = ""
+
+	// disable access log
+	cfg.AccessLog = ""
+
+	cfg.LMDB.MapSize = 20 * 1024 * 1024
+	cfg.LMDB.NoSync = true
+
+	return cfg
+}
+
+func (cfg *LevelDBConfig) Adjust() {
+	if cfg.CacheSize <= 0 {
+		cfg.CacheSize = 4 * 1024 * 1024
+	}
+
+	if cfg.BlockSize <= 0 {
+		cfg.BlockSize = 4 * 1024
+	}
+
+	if cfg.WriteBufferSize <= 0 {
+		cfg.WriteBufferSize = 4 * 1024 * 1024
+	}
+
+	if cfg.MaxOpenFiles < 1024 {
+		cfg.MaxOpenFiles = 1024
+	}
+}
+
+func (cfg *BinLogConfig) Adjust() {
+	if cfg.MaxFileSize <= 0 {
+		cfg.MaxFileSize = DefaultBinLogFileSize
+	} else if cfg.MaxFileSize > MaxBinLogFileSize {
+		cfg.MaxFileSize = MaxBinLogFileSize
+	}
+
+	if cfg.MaxFileNum <= 0 {
+		cfg.MaxFileNum = DefaultBinLogFileNum
+	} else if cfg.MaxFileNum > MaxBinLogFileNum {
+		cfg.MaxFileNum = MaxBinLogFileNum
+	}
+}

vendor/github.com/lunny/nodb/const.go

@@ -0,0 +1,98 @@
+package nodb
+
+import (
+	"errors"
+)
+
+const (
+	NoneType    byte = 0
+	KVType      byte = 1
+	HashType    byte = 2
+	HSizeType   byte = 3
+	ListType    byte = 4
+	LMetaType   byte = 5
+	ZSetType    byte = 6
+	ZSizeType   byte = 7
+	ZScoreType  byte = 8
+	BitType     byte = 9
+	BitMetaType byte = 10
+	SetType     byte = 11
+	SSizeType   byte = 12
+
+	maxDataType byte = 100
+
+	ExpTimeType byte = 101
+	ExpMetaType byte = 102
+)
+
+var (
+	TypeName = map[byte]string{
+		KVType:      "kv",
+		HashType:    "hash",
+		HSizeType:   "hsize",
+		ListType:    "list",
+		LMetaType:   "lmeta",
+		ZSetType:    "zset",
+		ZSizeType:   "zsize",
+		ZScoreType:  "zscore",
+		BitType:     "bit",
+		BitMetaType: "bitmeta",
+		SetType:     "set",
+		SSizeType:   "ssize",
+		ExpTimeType: "exptime",
+		ExpMetaType: "expmeta",
+	}
+)
+
+const (
+	defaultScanCount int = 10
+)
+
+var (
+	errKeySize        = errors.New("invalid key size")
+	errValueSize      = errors.New("invalid value size")
+	errHashFieldSize  = errors.New("invalid hash field size")
+	errSetMemberSize  = errors.New("invalid set member size")
+	errZSetMemberSize = errors.New("invalid zset member size")
+	errExpireValue    = errors.New("invalid expire value")
+)
+
+const (
+	//we don't support too many databases
+	MaxDBNumber uint8 = 16
+
+	//max key size
+	MaxKeySize int = 1024
+
+	//max hash field size
+	MaxHashFieldSize int = 1024
+
+	//max zset member size
+	MaxZSetMemberSize int = 1024
+
+	//max set member size
+	MaxSetMemberSize int = 1024
+
+	//max value size
+	MaxValueSize int = 10 * 1024 * 1024
+)
+
+var (
+	ErrScoreMiss = errors.New("zset score miss")
+)
+
+const (
+	BinLogTypeDeletion uint8 = 0x0
+	BinLogTypePut      uint8 = 0x1
+	BinLogTypeCommand  uint8 = 0x2
+)
+
+const (
+	DBAutoCommit    uint8 = 0x0
+	DBInTransaction uint8 = 0x1
+	DBInMulti       uint8 = 0x2
+)
+
+var (
+	Version = "0.1"
+)

vendor/github.com/lunny/nodb/doc.go

@@ -0,0 +1,61 @@
+// package nodb is a high performance embedded NoSQL.
+//
+// nodb supports various data structure like kv, list, hash and zset like redis.
+//
+// Other features include binlog replication, data with a limited time-to-live.
+//
+// Usage
+//
+// First create a nodb instance before use:
+//
+//  l := nodb.Open(cfg)
+//
+// cfg is a Config instance which contains configuration for nodb use,
+// like DataDir (root directory for nodb working to store data).
+//
+// After you create a nodb instance, you can select a DB to store you data:
+//
+//  db, _ := l.Select(0)
+//
+// DB must be selected by a index, nodb supports only 16 databases, so the index range is [0-15].
+//
+// KV
+//
+// KV is the most basic nodb type like any other key-value database.
+//
+//  err := db.Set(key, value)
+//  value, err := db.Get(key)
+//
+// List
+//
+// List is simply lists of values, sorted by insertion order.
+// You can push or pop value on the list head (left) or tail (right).
+//
+//  err := db.LPush(key, value1)
+//  err := db.RPush(key, value2)
+//  value1, err := db.LPop(key)
+//  value2, err := db.RPop(key)
+//
+// Hash
+//
+// Hash is a map between fields and values.
+//
+//  n, err := db.HSet(key, field1, value1)
+//  n, err := db.HSet(key, field2, value2)
+//  value1, err := db.HGet(key, field1)
+//  value2, err := db.HGet(key, field2)
+//
+// ZSet
+//
+// ZSet is a sorted collections of values.
+// Every member of zset is associated with score, a int64 value which used to sort, from smallest to greatest score.
+// Members are unique, but score may be same.
+//
+//  n, err := db.ZAdd(key, ScorePair{score1, member1}, ScorePair{score2, member2})
+//  ay, err := db.ZRangeByScore(key, minScore, maxScore, 0, -1)
+//
+// Binlog
+//
+// nodb supports binlog, so you can sync binlog to another server for replication. If you want to open binlog support, set UseBinLog to true in config.
+//
+package nodb

vendor/github.com/lunny/nodb/dump.go

@@ -0,0 +1,200 @@
+package nodb
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"io"
+	"os"
+
+	"github.com/siddontang/go-snappy/snappy"
+)
+
+//dump format
+// fileIndex(bigendian int64)|filePos(bigendian int64)
+// |keylen(bigendian int32)|key|valuelen(bigendian int32)|value......
+//
+//key and value are both compressed for fast transfer dump on network using snappy
+
+type BinLogAnchor struct {
+	LogFileIndex int64
+	LogPos       int64
+}
+
+func (m *BinLogAnchor) WriteTo(w io.Writer) error {
+	if err := binary.Write(w, binary.BigEndian, m.LogFileIndex); err != nil {
+		return err
+	}
+
+	if err := binary.Write(w, binary.BigEndian, m.LogPos); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *BinLogAnchor) ReadFrom(r io.Reader) error {
+	err := binary.Read(r, binary.BigEndian, &m.LogFileIndex)
+	if err != nil {
+		return err
+	}
+
+	err = binary.Read(r, binary.BigEndian, &m.LogPos)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (l *Nodb) DumpFile(path string) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return l.Dump(f)
+}
+
+func (l *Nodb) Dump(w io.Writer) error {
+	m := new(BinLogAnchor)
+
+	var err error
+
+	l.wLock.Lock()
+	defer l.wLock.Unlock()
+
+	if l.binlog != nil {
+		m.LogFileIndex = l.binlog.LogFileIndex()
+		m.LogPos = l.binlog.LogFilePos()
+	}
+
+	wb := bufio.NewWriterSize(w, 4096)
+	if err = m.WriteTo(wb); err != nil {
+		return err
+	}
+
+	it := l.ldb.NewIterator()
+	it.SeekToFirst()
+
+	compressBuf := make([]byte, 4096)
+
+	var key []byte
+	var value []byte
+	for ; it.Valid(); it.Next() {
+		key = it.RawKey()
+		value = it.RawValue()
+
+		if key, err = snappy.Encode(compressBuf, key); err != nil {
+			return err
+		}
+
+		if err = binary.Write(wb, binary.BigEndian, uint16(len(key))); err != nil {
+			return err
+		}
+
+		if _, err = wb.Write(key); err != nil {
+			return err
+		}
+
+		if value, err = snappy.Encode(compressBuf, value); err != nil {
+			return err
+		}
+
+		if err = binary.Write(wb, binary.BigEndian, uint32(len(value))); err != nil {
+			return err
+		}
+
+		if _, err = wb.Write(value); err != nil {
+			return err
+		}
+	}
+
+	if err = wb.Flush(); err != nil {
+		return err
+	}
+
+	compressBuf = nil
+
+	return nil
+}
+
+func (l *Nodb) LoadDumpFile(path string) (*BinLogAnchor, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	return l.LoadDump(f)
+}
+
+func (l *Nodb) LoadDump(r io.Reader) (*BinLogAnchor, error) {
+	l.wLock.Lock()
+	defer l.wLock.Unlock()
+
+	info := new(BinLogAnchor)
+
+	rb := bufio.NewReaderSize(r, 4096)
+
+	err := info.ReadFrom(rb)
+	if err != nil {
+		return nil, err
+	}
+
+	var keyLen uint16
+	var valueLen uint32
+
+	var keyBuf bytes.Buffer
+	var valueBuf bytes.Buffer
+
+	deKeyBuf := make([]byte, 4096)
+	deValueBuf := make([]byte, 4096)
+
+	var key, value []byte
+
+	for {
+		if err = binary.Read(rb, binary.BigEndian, &keyLen); err != nil && err != io.EOF {
+			return nil, err
+		} else if err == io.EOF {
+			break
+		}
+
+		if _, err = io.CopyN(&keyBuf, rb, int64(keyLen)); err != nil {
+			return nil, err
+		}
+
+		if key, err = snappy.Decode(deKeyBuf, keyBuf.Bytes()); err != nil {
+			return nil, err
+		}
+
+		if err = binary.Read(rb, binary.BigEndian, &valueLen); err != nil {
+			return nil, err
+		}
+
+		if _, err = io.CopyN(&valueBuf, rb, int64(valueLen)); err != nil {
+			return nil, err
+		}
+
+		if value, err = snappy.Decode(deValueBuf, valueBuf.Bytes()); err != nil {
+			return nil, err
+		}
+
+		if err = l.ldb.Put(key, value); err != nil {
+			return nil, err
+		}
+
+		keyBuf.Reset()
+		valueBuf.Reset()
+	}
+
+	deKeyBuf = nil
+	deValueBuf = nil
+
+	//if binlog enable, we will delete all binlogs and open a new one for handling simply
+	if l.binlog != nil {
+		l.binlog.PurgeAll()
+	}
+
+	return info, nil
+}

vendor/github.com/lunny/nodb/info.go

@@ -0,0 +1,24 @@
+package nodb
+
+// todo, add info
+
+// type Keyspace struct {
+// 	Kvs       int `json:"kvs"`
+// 	KvExpires int `json:"kv_expires"`
+
+// 	Lists       int `json:"lists"`
+// 	ListExpires int `json:"list_expires"`
+
+// 	Bitmaps       int `json:"bitmaps"`
+// 	BitmapExpires int `json:"bitmap_expires"`
+
+// 	ZSets       int `json:"zsets"`
+// 	ZSetExpires int `json:"zset_expires"`
+
+// 	Hashes      int `json:"hashes"`
+// 	HashExpires int `json:"hahsh_expires"`
+// }
+
+// type Info struct {
+// 	KeySpaces [MaxDBNumber]Keyspace
+// }

vendor/github.com/lunny/nodb/multi.go

@@ -0,0 +1,73 @@
+package nodb
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	ErrNestMulti = errors.New("nest multi not supported")
+	ErrMultiDone = errors.New("multi has been closed")
+)
+
+type Multi struct {
+	*DB
+}
+
+func (db *DB) IsInMulti() bool {
+	return db.status == DBInMulti
+}
+
+// begin a mutli to execute commands,
+// it will block any other write operations before you close the multi, unlike transaction, mutli can not rollback
+func (db *DB) Multi() (*Multi, error) {
+	if db.IsInMulti() {
+		return nil, ErrNestMulti
+	}
+
+	m := new(Multi)
+
+	m.DB = new(DB)
+	m.DB.status = DBInMulti
+
+	m.DB.l = db.l
+
+	m.l.wLock.Lock()
+
+	m.DB.sdb = db.sdb
+
+	m.DB.bucket = db.sdb
+
+	m.DB.index = db.index
+
+	m.DB.kvBatch = m.newBatch()
+	m.DB.listBatch = m.newBatch()
+	m.DB.hashBatch = m.newBatch()
+	m.DB.zsetBatch = m.newBatch()
+	m.DB.binBatch = m.newBatch()
+	m.DB.setBatch = m.newBatch()
+
+	return m, nil
+}
+
+func (m *Multi) newBatch() *batch {
+	return m.l.newBatch(m.bucket.NewWriteBatch(), &multiBatchLocker{}, nil)
+}
+
+func (m *Multi) Close() error {
+	if m.bucket == nil {
+		return ErrMultiDone
+	}
+	m.l.wLock.Unlock()
+	m.bucket = nil
+	return nil
+}
+
+func (m *Multi) Select(index int) error {
+	if index < 0 || index >= int(MaxDBNumber) {
+		return fmt.Errorf("invalid db index %d", index)
+	}
+
+	m.DB.index = uint8(index)
+	return nil
+}

vendor/github.com/lunny/nodb/nodb.go

@@ -0,0 +1,128 @@
+package nodb
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/lunny/log"
+	"github.com/lunny/nodb/config"
+	"github.com/lunny/nodb/store"
+)
+
+type Nodb struct {
+	cfg *config.Config
+
+	ldb *store.DB
+	dbs [MaxDBNumber]*DB
+
+	quit chan struct{}
+	jobs *sync.WaitGroup
+
+	binlog *BinLog
+
+	wLock      sync.RWMutex //allow one write at same time
+	commitLock sync.Mutex   //allow one write commit at same time
+}
+
+func Open(cfg *config.Config) (*Nodb, error) {
+	if len(cfg.DataDir) == 0 {
+		cfg.DataDir = config.DefaultDataDir
+	}
+
+	ldb, err := store.Open(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	l := new(Nodb)
+
+	l.quit = make(chan struct{})
+	l.jobs = new(sync.WaitGroup)
+
+	l.ldb = ldb
+
+	if cfg.BinLog.MaxFileNum > 0 && cfg.BinLog.MaxFileSize > 0 {
+		l.binlog, err = NewBinLog(cfg)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		l.binlog = nil
+	}
+
+	for i := uint8(0); i < MaxDBNumber; i++ {
+		l.dbs[i] = l.newDB(i)
+	}
+
+	l.activeExpireCycle()
+
+	return l, nil
+}
+
+func (l *Nodb) Close() {
+	close(l.quit)
+	l.jobs.Wait()
+
+	l.ldb.Close()
+
+	if l.binlog != nil {
+		l.binlog.Close()
+		l.binlog = nil
+	}
+}
+
+func (l *Nodb) Select(index int) (*DB, error) {
+	if index < 0 || index >= int(MaxDBNumber) {
+		return nil, fmt.Errorf("invalid db index %d", index)
+	}
+
+	return l.dbs[index], nil
+}
+
+func (l *Nodb) FlushAll() error {
+	for index, db := range l.dbs {
+		if _, err := db.FlushAll(); err != nil {
+			log.Error("flush db %d error %s", index, err.Error())
+		}
+	}
+
+	return nil
+}
+
+// very dangerous to use
+func (l *Nodb) DataDB() *store.DB {
+	return l.ldb
+}
+
+func (l *Nodb) activeExpireCycle() {
+	var executors []*elimination = make([]*elimination, len(l.dbs))
+	for i, db := range l.dbs {
+		executors[i] = db.newEliminator()
+	}
+
+	l.jobs.Add(1)
+	go func() {
+		tick := time.NewTicker(1 * time.Second)
+		end := false
+		done := make(chan struct{})
+		for !end {
+			select {
+			case <-tick.C:
+				go func() {
+					for _, eli := range executors {
+						eli.active()
+					}
+					done <- struct{}{}
+				}()
+				<-done
+			case <-l.quit:
+				end = true
+				break
+			}
+		}
+
+		tick.Stop()
+		l.jobs.Done()
+	}()
+}

vendor/github.com/lunny/nodb/nodb_db.go

@@ -0,0 +1,171 @@
+package nodb
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/lunny/nodb/store"
+)
+
+type ibucket interface {
+	Get(key []byte) ([]byte, error)
+
+	Put(key []byte, value []byte) error
+	Delete(key []byte) error
+
+	NewIterator() *store.Iterator
+
+	NewWriteBatch() store.WriteBatch
+
+	RangeIterator(min []byte, max []byte, rangeType uint8) *store.RangeLimitIterator
+	RevRangeIterator(min []byte, max []byte, rangeType uint8) *store.RangeLimitIterator
+	RangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *store.RangeLimitIterator
+	RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *store.RangeLimitIterator
+}
+
+type DB struct {
+	l *Nodb
+
+	sdb *store.DB
+
+	bucket ibucket
+
+	index uint8
+
+	kvBatch   *batch
+	listBatch *batch
+	hashBatch *batch
+	zsetBatch *batch
+	binBatch  *batch
+	setBatch  *batch
+
+	status uint8
+}
+
+func (l *Nodb) newDB(index uint8) *DB {
+	d := new(DB)
+
+	d.l = l
+
+	d.sdb = l.ldb
+
+	d.bucket = d.sdb
+
+	d.status = DBAutoCommit
+	d.index = index
+
+	d.kvBatch = d.newBatch()
+	d.listBatch = d.newBatch()
+	d.hashBatch = d.newBatch()
+	d.zsetBatch = d.newBatch()
+	d.binBatch = d.newBatch()
+	d.setBatch = d.newBatch()
+
+	return d
+}
+
+func (db *DB) newBatch() *batch {
+	return db.l.newBatch(db.bucket.NewWriteBatch(), &dbBatchLocker{l: &sync.Mutex{}, wrLock: &db.l.wLock}, nil)
+}
+
+func (db *DB) Index() int {
+	return int(db.index)
+}
+
+func (db *DB) IsAutoCommit() bool {
+	return db.status == DBAutoCommit
+}
+
+func (db *DB) FlushAll() (drop int64, err error) {
+	all := [...](func() (int64, error)){
+		db.flush,
+		db.lFlush,
+		db.hFlush,
+		db.zFlush,
+		db.bFlush,
+		db.sFlush}
+
+	for _, flush := range all {
+		if n, e := flush(); e != nil {
+			err = e
+			return
+		} else {
+			drop += n
+		}
+	}
+
+	return
+}
+
+func (db *DB) newEliminator() *elimination {
+	eliminator := newEliminator(db)
+
+	eliminator.regRetireContext(KVType, db.kvBatch, db.delete)
+	eliminator.regRetireContext(ListType, db.listBatch, db.lDelete)
+	eliminator.regRetireContext(HashType, db.hashBatch, db.hDelete)
+	eliminator.regRetireContext(ZSetType, db.zsetBatch, db.zDelete)
+	eliminator.regRetireContext(BitType, db.binBatch, db.bDelete)
+	eliminator.regRetireContext(SetType, db.setBatch, db.sDelete)
+
+	return eliminator
+}
+
+func (db *DB) flushRegion(t *batch, minKey []byte, maxKey []byte) (drop int64, err error) {
+	it := db.bucket.RangeIterator(minKey, maxKey, store.RangeROpen)
+	for ; it.Valid(); it.Next() {
+		t.Delete(it.RawKey())
+		drop++
+		if drop&1023 == 0 {
+			if err = t.Commit(); err != nil {
+				return
+			}
+		}
+	}
+	it.Close()
+	return
+}
+
+func (db *DB) flushType(t *batch, dataType byte) (drop int64, err error) {
+	var deleteFunc func(t *batch, key []byte) int64
+	var metaDataType byte
+	switch dataType {
+	case KVType:
+		deleteFunc = db.delete
+		metaDataType = KVType
+	case ListType:
+		deleteFunc = db.lDelete
+		metaDataType = LMetaType
+	case HashType:
+		deleteFunc = db.hDelete
+		metaDataType = HSizeType
+	case ZSetType:
+		deleteFunc = db.zDelete
+		metaDataType = ZSizeType
+	case BitType:
+		deleteFunc = db.bDelete
+		metaDataType = BitMetaType
+	case SetType:
+		deleteFunc = db.sDelete
+		metaDataType = SSizeType
+	default:
+		return 0, fmt.Errorf("invalid data type: %s", TypeName[dataType])
+	}
+
+	var keys [][]byte
+	keys, err = db.scan(metaDataType, nil, 1024, false, "")
+	for len(keys) != 0 || err != nil {
+		for _, key := range keys {
+			deleteFunc(t, key)
+			db.rmExpire(t, dataType, key)
+
+		}
+
+		if err = t.Commit(); err != nil {
+			return
+		} else {
+			drop += int64(len(keys))
+		}
+		keys, err = db.scan(metaDataType, nil, 1024, false, "")
+	}
+	return
+}

vendor/github.com/lunny/nodb/replication.go

@@ -0,0 +1,312 @@
+package nodb
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"io"
+	"os"
+	"time"
+
+	"github.com/lunny/log"
+	"github.com/lunny/nodb/store/driver"
+)
+
+const (
+	maxReplBatchNum = 100
+	maxReplLogSize  = 1 * 1024 * 1024
+)
+
+var (
+	ErrSkipEvent = errors.New("skip to next event")
+)
+
+var (
+	errInvalidBinLogEvent = errors.New("invalid binglog event")
+	errInvalidBinLogFile  = errors.New("invalid binlog file")
+)
+
+type replBatch struct {
+	wb     driver.IWriteBatch
+	events [][]byte
+	l      *Nodb
+
+	lastHead *BinLogHead
+}
+
+func (b *replBatch) Commit() error {
+	b.l.commitLock.Lock()
+	defer b.l.commitLock.Unlock()
+
+	err := b.wb.Commit()
+	if err != nil {
+		b.Rollback()
+		return err
+	}
+
+	if b.l.binlog != nil {
+		if err = b.l.binlog.Log(b.events...); err != nil {
+			b.Rollback()
+			return err
+		}
+	}
+
+	b.events = [][]byte{}
+	b.lastHead = nil
+
+	return nil
+}
+
+func (b *replBatch) Rollback() error {
+	b.wb.Rollback()
+	b.events = [][]byte{}
+	b.lastHead = nil
+	return nil
+}
+
+func (l *Nodb) replicateEvent(b *replBatch, event []byte) error {
+	if len(event) == 0 {
+		return errInvalidBinLogEvent
+	}
+
+	b.events = append(b.events, event)
+
+	logType := uint8(event[0])
+	switch logType {
+	case BinLogTypePut:
+		return l.replicatePutEvent(b, event)
+	case BinLogTypeDeletion:
+		return l.replicateDeleteEvent(b, event)
+	default:
+		return errInvalidBinLogEvent
+	}
+}
+
+func (l *Nodb) replicatePutEvent(b *replBatch, event []byte) error {
+	key, value, err := decodeBinLogPut(event)
+	if err != nil {
+		return err
+	}
+
+	b.wb.Put(key, value)
+
+	return nil
+}
+
+func (l *Nodb) replicateDeleteEvent(b *replBatch, event []byte) error {
+	key, err := decodeBinLogDelete(event)
+	if err != nil {
+		return err
+	}
+
+	b.wb.Delete(key)
+
+	return nil
+}
+
+func ReadEventFromReader(rb io.Reader, f func(head *BinLogHead, event []byte) error) error {
+	head := &BinLogHead{}
+	var err error
+
+	for {
+		if err = head.Read(rb); err != nil {
+			if err == io.EOF {
+				break
+			} else {
+				return err
+			}
+		}
+
+		var dataBuf bytes.Buffer
+
+		if _, err = io.CopyN(&dataBuf, rb, int64(head.PayloadLen)); err != nil {
+			return err
+		}
+
+		err = f(head, dataBuf.Bytes())
+		if err != nil && err != ErrSkipEvent {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (l *Nodb) ReplicateFromReader(rb io.Reader) error {
+	b := new(replBatch)
+
+	b.wb = l.ldb.NewWriteBatch()
+	b.l = l
+
+	f := func(head *BinLogHead, event []byte) error {
+		if b.lastHead == nil {
+			b.lastHead = head
+		} else if !b.lastHead.InSameBatch(head) {
+			if err := b.Commit(); err != nil {
+				log.Fatal("replication error %s, skip to next", err.Error())
+				return ErrSkipEvent
+			}
+			b.lastHead = head
+		}
+
+		err := l.replicateEvent(b, event)
+		if err != nil {
+			log.Fatal("replication error %s, skip to next", err.Error())
+			return ErrSkipEvent
+		}
+		return nil
+	}
+
+	err := ReadEventFromReader(rb, f)
+	if err != nil {
+		b.Rollback()
+		return err
+	}
+	return b.Commit()
+}
+
+func (l *Nodb) ReplicateFromData(data []byte) error {
+	rb := bytes.NewReader(data)
+
+	err := l.ReplicateFromReader(rb)
+
+	return err
+}
+
+func (l *Nodb) ReplicateFromBinLog(filePath string) error {
+	f, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+
+	rb := bufio.NewReaderSize(f, 4096)
+
+	err = l.ReplicateFromReader(rb)
+
+	f.Close()
+
+	return err
+}
+
+// try to read events, if no events read, try to wait the new event singal until timeout seconds
+func (l *Nodb) ReadEventsToTimeout(info *BinLogAnchor, w io.Writer, timeout int) (n int, err error) {
+	lastIndex := info.LogFileIndex
+	lastPos := info.LogPos
+
+	n = 0
+	if l.binlog == nil {
+		//binlog not supported
+		info.LogFileIndex = 0
+		info.LogPos = 0
+		return
+	}
+
+	n, err = l.ReadEventsTo(info, w)
+	if err == nil && info.LogFileIndex == lastIndex && info.LogPos == lastPos {
+		//no events read
+		select {
+		case <-l.binlog.Wait():
+		case <-time.After(time.Duration(timeout) * time.Second):
+		}
+		return l.ReadEventsTo(info, w)
+	}
+	return
+}
+
+func (l *Nodb) ReadEventsTo(info *BinLogAnchor, w io.Writer) (n int, err error) {
+	n = 0
+	if l.binlog == nil {
+		//binlog not supported
+		info.LogFileIndex = 0
+		info.LogPos = 0
+		return
+	}
+
+	index := info.LogFileIndex
+	offset := info.LogPos
+
+	filePath := l.binlog.FormatLogFilePath(index)
+
+	var f *os.File
+	f, err = os.Open(filePath)
+	if os.IsNotExist(err) {
+		lastIndex := l.binlog.LogFileIndex()
+
+		if index == lastIndex {
+			//no binlog at all
+			info.LogPos = 0
+		} else {
+			//slave binlog info had lost
+			info.LogFileIndex = -1
+		}
+	}
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			err = nil
+		}
+		return
+	}
+
+	defer f.Close()
+
+	var fileSize int64
+	st, _ := f.Stat()
+	fileSize = st.Size()
+
+	if fileSize == info.LogPos {
+		return
+	}
+
+	if _, err = f.Seek(offset, os.SEEK_SET); err != nil {
+		//may be invliad seek offset
+		return
+	}
+
+	var lastHead *BinLogHead = nil
+
+	head := &BinLogHead{}
+
+	batchNum := 0
+
+	for {
+		if err = head.Read(f); err != nil {
+			if err == io.EOF {
+				//we will try to use next binlog
+				if index < l.binlog.LogFileIndex() {
+					info.LogFileIndex += 1
+					info.LogPos = 0
+				}
+				err = nil
+				return
+			} else {
+				return
+			}
+
+		}
+
+		if lastHead == nil {
+			lastHead = head
+			batchNum++
+		} else if !lastHead.InSameBatch(head) {
+			lastHead = head
+			batchNum++
+			if batchNum > maxReplBatchNum || n > maxReplLogSize {
+				return
+			}
+		}
+
+		if err = head.Write(w); err != nil {
+			return
+		}
+
+		if _, err = io.CopyN(w, f, int64(head.PayloadLen)); err != nil {
+			return
+		}
+
+		n += (head.Len() + int(head.PayloadLen))
+		info.LogPos = info.LogPos + int64(head.Len()) + int64(head.PayloadLen)
+	}
+
+	return
+}

vendor/github.com/lunny/nodb/scan.go

@@ -0,0 +1,144 @@
+package nodb
+
+import (
+	"bytes"
+	"errors"
+	"regexp"
+
+	"github.com/lunny/nodb/store"
+)
+
+var errDataType = errors.New("error data type")
+var errMetaKey = errors.New("error meta key")
+
+// Seek search the prefix key
+func (db *DB) Seek(key []byte) (*store.Iterator, error) {
+	return db.seek(KVType, key)
+}
+
+func (db *DB) seek(dataType byte, key []byte) (*store.Iterator, error) {
+	var minKey []byte
+	var err error
+
+	if len(key) > 0 {
+		if err = checkKeySize(key); err != nil {
+			return nil, err
+		}
+		if minKey, err = db.encodeMetaKey(dataType, key); err != nil {
+			return nil, err
+		}
+
+	} else {
+		if minKey, err = db.encodeMinKey(dataType); err != nil {
+			return nil, err
+		}
+	}
+
+	it := db.bucket.NewIterator()
+	it.Seek(minKey)
+	return it, nil
+}
+
+func (db *DB) MaxKey() ([]byte, error) {
+	return db.encodeMaxKey(KVType)
+}
+
+func (db *DB) Key(it *store.Iterator) ([]byte, error) {
+	return db.decodeMetaKey(KVType, it.Key())
+}
+
+func (db *DB) scan(dataType byte, key []byte, count int, inclusive bool, match string) ([][]byte, error) {
+	var minKey, maxKey []byte
+	var err error
+	var r *regexp.Regexp
+
+	if len(match) > 0 {
+		if r, err = regexp.Compile(match); err != nil {
+			return nil, err
+		}
+	}
+
+	if len(key) > 0 {
+		if err = checkKeySize(key); err != nil {
+			return nil, err
+		}
+		if minKey, err = db.encodeMetaKey(dataType, key); err != nil {
+			return nil, err
+		}
+
+	} else {
+		if minKey, err = db.encodeMinKey(dataType); err != nil {
+			return nil, err
+		}
+	}
+
+	if maxKey, err = db.encodeMaxKey(dataType); err != nil {
+		return nil, err
+	}
+
+	if count <= 0 {
+		count = defaultScanCount
+	}
+
+	v := make([][]byte, 0, count)
+
+	it := db.bucket.NewIterator()
+	it.Seek(minKey)
+
+	if !inclusive {
+		if it.Valid() && bytes.Equal(it.RawKey(), minKey) {
+			it.Next()
+		}
+	}
+
+	for i := 0; it.Valid() && i < count && bytes.Compare(it.RawKey(), maxKey) < 0; it.Next() {
+		if k, err := db.decodeMetaKey(dataType, it.Key()); err != nil {
+			continue
+		} else if r != nil && !r.Match(k) {
+			continue
+		} else {
+			v = append(v, k)
+			i++
+		}
+	}
+	it.Close()
+	return v, nil
+}
+
+func (db *DB) encodeMinKey(dataType byte) ([]byte, error) {
+	return db.encodeMetaKey(dataType, nil)
+}
+
+func (db *DB) encodeMaxKey(dataType byte) ([]byte, error) {
+	k, err := db.encodeMetaKey(dataType, nil)
+	if err != nil {
+		return nil, err
+	}
+	k[len(k)-1] = dataType + 1
+	return k, nil
+}
+
+func (db *DB) encodeMetaKey(dataType byte, key []byte) ([]byte, error) {
+	switch dataType {
+	case KVType:
+		return db.encodeKVKey(key), nil
+	case LMetaType:
+		return db.lEncodeMetaKey(key), nil
+	case HSizeType:
+		return db.hEncodeSizeKey(key), nil
+	case ZSizeType:
+		return db.zEncodeSizeKey(key), nil
+	case BitMetaType:
+		return db.bEncodeMetaKey(key), nil
+	case SSizeType:
+		return db.sEncodeSizeKey(key), nil
+	default:
+		return nil, errDataType
+	}
+}
+func (db *DB) decodeMetaKey(dataType byte, ek []byte) ([]byte, error) {
+	if len(ek) < 2 || ek[0] != db.index || ek[1] != dataType {
+		return nil, errMetaKey
+	}
+	return ek[2:], nil
+}

vendor/github.com/lunny/nodb/store/db.go

@@ -0,0 +1,61 @@
+package store
+
+import (
+	"github.com/lunny/nodb/store/driver"
+)
+
+type DB struct {
+	driver.IDB
+}
+
+func (db *DB) NewIterator() *Iterator {
+	it := new(Iterator)
+	it.it = db.IDB.NewIterator()
+
+	return it
+}
+
+func (db *DB) NewWriteBatch() WriteBatch {
+	return db.IDB.NewWriteBatch()
+}
+
+func (db *DB) NewSnapshot() (*Snapshot, error) {
+	var err error
+	s := &Snapshot{}
+	if s.ISnapshot, err = db.IDB.NewSnapshot(); err != nil {
+		return nil, err
+	}
+
+	return s, nil
+}
+
+func (db *DB) RangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator {
+	return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1})
+}
+
+func (db *DB) RevRangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator {
+	return NewRevRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1})
+}
+
+//count < 0, unlimit.
+//
+//offset must >= 0, if < 0, will get nothing.
+func (db *DB) RangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *RangeLimitIterator {
+	return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count})
+}
+
+//count < 0, unlimit.
+//
+//offset must >= 0, if < 0, will get nothing.
+func (db *DB) RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *RangeLimitIterator {
+	return NewRevRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count})
+}
+
+func (db *DB) Begin() (*Tx, error) {
+	tx, err := db.IDB.Begin()
+	if err != nil {
+		return nil, err
+	}
+
+	return &Tx{tx}, nil
+}

vendor/github.com/lunny/nodb/store/driver/batch.go

@@ -0,0 +1,39 @@
+package driver
+
+type BatchPuter interface {
+	BatchPut([]Write) error
+}
+
+type Write struct {
+	Key   []byte
+	Value []byte
+}
+
+type WriteBatch struct {
+	batch BatchPuter
+	wb    []Write
+}
+
+func (w *WriteBatch) Put(key, value []byte) {
+	if value == nil {
+		value = []byte{}
+	}
+	w.wb = append(w.wb, Write{key, value})
+}
+
+func (w *WriteBatch) Delete(key []byte) {
+	w.wb = append(w.wb, Write{key, nil})
+}
+
+func (w *WriteBatch) Commit() error {
+	return w.batch.BatchPut(w.wb)
+}
+
+func (w *WriteBatch) Rollback() error {
+	w.wb = w.wb[0:0]
+	return nil
+}
+
+func NewWriteBatch(puter BatchPuter) IWriteBatch {
+	return &WriteBatch{puter, []Write{}}
+}

vendor/github.com/lunny/nodb/store/driver/driver.go

@@ -0,0 +1,67 @@
+package driver
+
+import (
+	"errors"
+)
+
+var (
+	ErrTxSupport = errors.New("transaction is not supported")
+)
+
+type IDB interface {
+	Close() error
+
+	Get(key []byte) ([]byte, error)
+
+	Put(key []byte, value []byte) error
+	Delete(key []byte) error
+
+	NewIterator() IIterator
+
+	NewWriteBatch() IWriteBatch
+
+	NewSnapshot() (ISnapshot, error)
+
+	Begin() (Tx, error)
+}
+
+type ISnapshot interface {
+	Get(key []byte) ([]byte, error)
+	NewIterator() IIterator
+	Close()
+}
+
+type IIterator interface {
+	Close() error
+
+	First()
+	Last()
+	Seek(key []byte)
+
+	Next()
+	Prev()
+
+	Valid() bool
+
+	Key() []byte
+	Value() []byte
+}
+
+type IWriteBatch interface {
+	Put(key []byte, value []byte)
+	Delete(key []byte)
+	Commit() error
+	Rollback() error
+}
+
+type Tx interface {
+	Get(key []byte) ([]byte, error)
+	Put(key []byte, value []byte) error
+	Delete(key []byte) error
+
+	NewIterator() IIterator
+	NewWriteBatch() IWriteBatch
+
+	Commit() error
+	Rollback() error
+}

vendor/github.com/lunny/nodb/store/driver/store.go

@@ -0,0 +1,46 @@
+package driver
+
+import (
+	"fmt"
+
+	"github.com/lunny/nodb/config"
+)
+
+type Store interface {
+	String() string
+	Open(path string, cfg *config.Config) (IDB, error)
+	Repair(path string, cfg *config.Config) error
+}
+
+var dbs = map[string]Store{}
+
+func Register(s Store) {
+	name := s.String()
+	if _, ok := dbs[name]; ok {
+		panic(fmt.Errorf("store %s is registered", s))
+	}
+
+	dbs[name] = s
+}
+
+func ListStores() []string {
+	s := []string{}
+	for k, _ := range dbs {
+		s = append(s, k)
+	}
+
+	return s
+}
+
+func GetStore(cfg *config.Config) (Store, error) {
+	if len(cfg.DBName) == 0 {
+		cfg.DBName = config.DefaultDBName
+	}
+
+	s, ok := dbs[cfg.DBName]
+	if !ok {
+		return nil, fmt.Errorf("store %s is not registered", cfg.DBName)
+	}
+
+	return s, nil
+}

vendor/github.com/lunny/nodb/store/goleveldb/batch.go

@@ -0,0 +1,27 @@
+package goleveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type WriteBatch struct {
+	db     *DB
+	wbatch *leveldb.Batch
+}
+
+func (w *WriteBatch) Put(key, value []byte) {
+	w.wbatch.Put(key, value)
+}
+
+func (w *WriteBatch) Delete(key []byte) {
+	w.wbatch.Delete(key)
+}
+
+func (w *WriteBatch) Commit() error {
+	return w.db.db.Write(w.wbatch, nil)
+}
+
+func (w *WriteBatch) Rollback() error {
+	w.wbatch.Reset()
+	return nil
+}

vendor/github.com/lunny/nodb/store/goleveldb/const.go

@@ -0,0 +1,4 @@
+package goleveldb
+
+const DBName = "goleveldb"
+const MemDBName = "memory"

vendor/github.com/lunny/nodb/store/goleveldb/db.go

@@ -0,0 +1,187 @@
+package goleveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/cache"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+
+	"github.com/lunny/nodb/config"
+	"github.com/lunny/nodb/store/driver"
+
+	"os"
+)
+
+const defaultFilterBits int = 10
+
+type Store struct {
+}
+
+func (s Store) String() string {
+	return DBName
+}
+
+type MemStore struct {
+}
+
+func (s MemStore) String() string {
+	return MemDBName
+}
+
+type DB struct {
+	path string
+
+	cfg *config.LevelDBConfig
+
+	db *leveldb.DB
+
+	opts *opt.Options
+
+	iteratorOpts *opt.ReadOptions
+
+	cache cache.Cache
+
+	filter filter.Filter
+}
+
+func (s Store) Open(path string, cfg *config.Config) (driver.IDB, error) {
+	if err := os.MkdirAll(path, os.ModePerm); err != nil {
+		return nil, err
+	}
+
+	db := new(DB)
+	db.path = path
+	db.cfg = &cfg.LevelDB
+
+	db.initOpts()
+
+	var err error
+	db.db, err = leveldb.OpenFile(db.path, db.opts)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return db, nil
+}
+
+func (s Store) Repair(path string, cfg *config.Config) error {
+	db, err := leveldb.RecoverFile(path, newOptions(&cfg.LevelDB))
+	if err != nil {
+		return err
+	}
+
+	db.Close()
+	return nil
+}
+
+func (s MemStore) Open(path string, cfg *config.Config) (driver.IDB, error) {
+	db := new(DB)
+	db.path = path
+	db.cfg = &cfg.LevelDB
+
+	db.initOpts()
+
+	var err error
+	db.db, err = leveldb.Open(storage.NewMemStorage(), db.opts)
+	if err != nil {
+		return nil, err
+	}
+
+	return db, nil
+}
+
+func (s MemStore) Repair(path string, cfg *config.Config) error {
+	return nil
+}
+
+func (db *DB) initOpts() {
+	db.opts = newOptions(db.cfg)
+
+	db.iteratorOpts = &opt.ReadOptions{}
+	db.iteratorOpts.DontFillCache = true
+}
+
+func newOptions(cfg *config.LevelDBConfig) *opt.Options {
+	opts := &opt.Options{}
+	opts.ErrorIfMissing = false
+
+	cfg.Adjust()
+
+	//opts.BlockCacher = cache.NewLRU(cfg.CacheSize)
+	opts.BlockCacheCapacity = cfg.CacheSize
+
+	//we must use bloomfilter
+	opts.Filter = filter.NewBloomFilter(defaultFilterBits)
+
+	if !cfg.Compression {
+		opts.Compression = opt.NoCompression
+	} else {
+		opts.Compression = opt.SnappyCompression
+	}
+
+	opts.BlockSize = cfg.BlockSize
+	opts.WriteBuffer = cfg.WriteBufferSize
+
+	return opts
+}
+
+func (db *DB) Close() error {
+	return db.db.Close()
+}
+
+func (db *DB) Put(key, value []byte) error {
+	return db.db.Put(key, value, nil)
+}
+
+func (db *DB) Get(key []byte) ([]byte, error) {
+	v, err := db.db.Get(key, nil)
+	if err == leveldb.ErrNotFound {
+		return nil, nil
+	}
+	return v, nil
+}
+
+func (db *DB) Delete(key []byte) error {
+	return db.db.Delete(key, nil)
+}
+
+func (db *DB) NewWriteBatch() driver.IWriteBatch {
+	wb := &WriteBatch{
+		db:     db,
+		wbatch: new(leveldb.Batch),
+	}
+	return wb
+}
+
+func (db *DB) NewIterator() driver.IIterator {
+	it := &Iterator{
+		db.db.NewIterator(nil, db.iteratorOpts),
+	}
+
+	return it
+}
+
+func (db *DB) Begin() (driver.Tx, error) {
+	return nil, driver.ErrTxSupport
+}
+
+func (db *DB) NewSnapshot() (driver.ISnapshot, error) {
+	snapshot, err := db.db.GetSnapshot()
+	if err != nil {
+		return nil, err
+	}
+
+	s := &Snapshot{
+		db:  db,
+		snp: snapshot,
+	}
+
+	return s, nil
+}
+
+func init() {
+	driver.Register(Store{})
+	driver.Register(MemStore{})
+}

vendor/github.com/lunny/nodb/store/goleveldb/iterator.go

@@ -0,0 +1,49 @@
+package goleveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+)
+
+type Iterator struct {
+	it iterator.Iterator
+}
+
+func (it *Iterator) Key() []byte {
+	return it.it.Key()
+}
+
+func (it *Iterator) Value() []byte {
+	return it.it.Value()
+}
+
+func (it *Iterator) Close() error {
+	if it.it != nil {
+		it.it.Release()
+		it.it = nil
+	}
+	return nil
+}
+
+func (it *Iterator) Valid() bool {
+	return it.it.Valid()
+}
+
+func (it *Iterator) Next() {
+	it.it.Next()
+}
+
+func (it *Iterator) Prev() {
+	it.it.Prev()
+}
+
+func (it *Iterator) First() {
+	it.it.First()
+}
+
+func (it *Iterator) Last() {
+	it.it.Last()
+}
+
+func (it *Iterator) Seek(key []byte) {
+	it.it.Seek(key)
+}

vendor/github.com/lunny/nodb/store/goleveldb/snapshot.go

@@ -0,0 +1,26 @@
+package goleveldb
+
+import (
+	"github.com/lunny/nodb/store/driver"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type Snapshot struct {
+	db  *DB
+	snp *leveldb.Snapshot
+}
+
+func (s *Snapshot) Get(key []byte) ([]byte, error) {
+	return s.snp.Get(key, s.db.iteratorOpts)
+}
+
+func (s *Snapshot) NewIterator() driver.IIterator {
+	it := &Iterator{
+		s.snp.NewIterator(nil, s.db.iteratorOpts),
+	}
+	return it
+}
+
+func (s *Snapshot) Close() {
+	s.snp.Release()
+}

vendor/github.com/lunny/nodb/store/iterator.go

@@ -0,0 +1,327 @@
+package store
+
+import (
+	"bytes"
+
+	"github.com/lunny/nodb/store/driver"
+)
+
+const (
+	IteratorForward  uint8 = 0
+	IteratorBackward uint8 = 1
+)
+
+const (
+	RangeClose uint8 = 0x00
+	RangeLOpen uint8 = 0x01
+	RangeROpen uint8 = 0x10
+	RangeOpen  uint8 = 0x11
+)
+
+// min must less or equal than max
+//
+// range type:
+//
+//  close: [min, max]
+//  open: (min, max)
+//  lopen: (min, max]
+//  ropen: [min, max)
+//
+type Range struct {
+	Min []byte
+	Max []byte
+
+	Type uint8
+}
+
+type Limit struct {
+	Offset int
+	Count  int
+}
+
+type Iterator struct {
+	it driver.IIterator
+}
+
+// Returns a copy of key.
+func (it *Iterator) Key() []byte {
+	k := it.it.Key()
+	if k == nil {
+		return nil
+	}
+
+	return append([]byte{}, k...)
+}
+
+// Returns a copy of value.
+func (it *Iterator) Value() []byte {
+	v := it.it.Value()
+	if v == nil {
+		return nil
+	}
+
+	return append([]byte{}, v...)
+}
+
+// Returns a reference of key.
+// you must be careful that it will be changed after next iterate.
+func (it *Iterator) RawKey() []byte {
+	return it.it.Key()
+}
+
+// Returns a reference of value.
+// you must be careful that it will be changed after next iterate.
+func (it *Iterator) RawValue() []byte {
+	return it.it.Value()
+}
+
+// Copy key to b, if b len is small or nil, returns a new one.
+func (it *Iterator) BufKey(b []byte) []byte {
+	k := it.RawKey()
+	if k == nil {
+		return nil
+	}
+	if b == nil {
+		b = []byte{}
+	}
+
+	b = b[0:0]
+	return append(b, k...)
+}
+
+// Copy value to b, if b len is small or nil, returns a new one.
+func (it *Iterator) BufValue(b []byte) []byte {
+	v := it.RawValue()
+	if v == nil {
+		return nil
+	}
+
+	if b == nil {
+		b = []byte{}
+	}
+
+	b = b[0:0]
+	return append(b, v...)
+}
+
+func (it *Iterator) Close() {
+	if it.it != nil {
+		it.it.Close()
+		it.it = nil
+	}
+}
+
+func (it *Iterator) Valid() bool {
+	return it.it.Valid()
+}
+
+func (it *Iterator) Next() {
+	it.it.Next()
+}
+
+func (it *Iterator) Prev() {
+	it.it.Prev()
+}
+
+func (it *Iterator) SeekToFirst() {
+	it.it.First()
+}
+
+func (it *Iterator) SeekToLast() {
+	it.it.Last()
+}
+
+func (it *Iterator) Seek(key []byte) {
+	it.it.Seek(key)
+}
+
+// Finds by key, if not found, nil returns.
+func (it *Iterator) Find(key []byte) []byte {
+	it.Seek(key)
+	if it.Valid() {
+		k := it.RawKey()
+		if k == nil {
+			return nil
+		} else if bytes.Equal(k, key) {
+			return it.Value()
+		}
+	}
+
+	return nil
+}
+
+// Finds by key, if not found, nil returns, else a reference of value returns.
+// you must be careful that it will be changed after next iterate.
+func (it *Iterator) RawFind(key []byte) []byte {
+	it.Seek(key)
+	if it.Valid() {
+		k := it.RawKey()
+		if k == nil {
+			return nil
+		} else if bytes.Equal(k, key) {
+			return it.RawValue()
+		}
+	}
+
+	return nil
+}
+
+type RangeLimitIterator struct {
+	it *Iterator
+
+	r *Range
+	l *Limit
+
+	step int
+
+	//0 for IteratorForward, 1 for IteratorBackward
+	direction uint8
+}
+
+func (it *RangeLimitIterator) Key() []byte {
+	return it.it.Key()
+}
+
+func (it *RangeLimitIterator) Value() []byte {
+	return it.it.Value()
+}
+
+func (it *RangeLimitIterator) RawKey() []byte {
+	return it.it.RawKey()
+}
+
+func (it *RangeLimitIterator) RawValue() []byte {
+	return it.it.RawValue()
+}
+
+func (it *RangeLimitIterator) BufKey(b []byte) []byte {
+	return it.it.BufKey(b)
+}
+
+func (it *RangeLimitIterator) BufValue(b []byte) []byte {
+	return it.it.BufValue(b)
+}
+
+func (it *RangeLimitIterator) Valid() bool {
+	if it.l.Offset < 0 {
+		return false
+	} else if !it.it.Valid() {
+		return false
+	} else if it.l.Count >= 0 && it.step >= it.l.Count {
+		return false
+	}
+
+	if it.direction == IteratorForward {
+		if it.r.Max != nil {
+			r := bytes.Compare(it.it.RawKey(), it.r.Max)
+			if it.r.Type&RangeROpen > 0 {
+				return !(r >= 0)
+			} else {
+				return !(r > 0)
+			}
+		}
+	} else {
+		if it.r.Min != nil {
+			r := bytes.Compare(it.it.RawKey(), it.r.Min)
+			if it.r.Type&RangeLOpen > 0 {
+				return !(r <= 0)
+			} else {
+				return !(r < 0)
+			}
+		}
+	}
+
+	return true
+}
+
+func (it *RangeLimitIterator) Next() {
+	it.step++
+
+	if it.direction == IteratorForward {
+		it.it.Next()
+	} else {
+		it.it.Prev()
+	}
+}
+
+func (it *RangeLimitIterator) Close() {
+	it.it.Close()
+}
+
+func NewRangeLimitIterator(i *Iterator, r *Range, l *Limit) *RangeLimitIterator {
+	return rangeLimitIterator(i, r, l, IteratorForward)
+}
+
+func NewRevRangeLimitIterator(i *Iterator, r *Range, l *Limit) *RangeLimitIterator {
+	return rangeLimitIterator(i, r, l, IteratorBackward)
+}
+
+func NewRangeIterator(i *Iterator, r *Range) *RangeLimitIterator {
+	return rangeLimitIterator(i, r, &Limit{0, -1}, IteratorForward)
+}
+
+func NewRevRangeIterator(i *Iterator, r *Range) *RangeLimitIterator {
+	return rangeLimitIterator(i, r, &Limit{0, -1}, IteratorBackward)
+}
+
+func rangeLimitIterator(i *Iterator, r *Range, l *Limit, direction uint8) *RangeLimitIterator {
+	it := new(RangeLimitIterator)
+
+	it.it = i
+
+	it.r = r
+	it.l = l
+	it.direction = direction
+
+	it.step = 0
+
+	if l.Offset < 0 {
+		return it
+	}
+
+	if direction == IteratorForward {
+		if r.Min == nil {
+			it.it.SeekToFirst()
+		} else {
+			it.it.Seek(r.Min)
+
+			if r.Type&RangeLOpen > 0 {
+				if it.it.Valid() && bytes.Equal(it.it.RawKey(), r.Min) {
+					it.it.Next()
+				}
+			}
+		}
+	} else {
+		if r.Max == nil {
+			it.it.SeekToLast()
+		} else {
+			it.it.Seek(r.Max)
+
+			if !it.it.Valid() {
+				it.it.SeekToLast()
+			} else {
+				if !bytes.Equal(it.it.RawKey(), r.Max) {
+					it.it.Prev()
+				}
+			}
+
+			if r.Type&RangeROpen > 0 {
+				if it.it.Valid() && bytes.Equal(it.it.RawKey(), r.Max) {
+					it.it.Prev()
+				}
+			}
+		}
+	}
+
+	for i := 0; i < l.Offset; i++ {
+		if it.it.Valid() {
+			if it.direction == IteratorForward {
+				it.it.Next()
+			} else {
+				it.it.Prev()
+			}
+		}
+	}
+
+	return it
+}

vendor/github.com/lunny/nodb/store/snapshot.go

@@ -0,0 +1,16 @@
+package store
+
+import (
+	"github.com/lunny/nodb/store/driver"
+)
+
+type Snapshot struct {
+	driver.ISnapshot
+}
+
+func (s *Snapshot) NewIterator() *Iterator {
+	it := new(Iterator)
+	it.it = s.ISnapshot.NewIterator()
+
+	return it
+}

vendor/github.com/lunny/nodb/store/store.go

@@ -0,0 +1,51 @@
+package store
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"github.com/lunny/nodb/config"
+	"github.com/lunny/nodb/store/driver"
+
+	_ "github.com/lunny/nodb/store/goleveldb"
+)
+
+func getStorePath(cfg *config.Config) string {
+	return path.Join(cfg.DataDir, fmt.Sprintf("%s_data", cfg.DBName))
+}
+
+func Open(cfg *config.Config) (*DB, error) {
+	s, err := driver.GetStore(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	path := getStorePath(cfg)
+
+	if err := os.MkdirAll(path, os.ModePerm); err != nil {
+		return nil, err
+	}
+
+	idb, err := s.Open(path, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	db := &DB{idb}
+
+	return db, nil
+}
+
+func Repair(cfg *config.Config) error {
+	s, err := driver.GetStore(cfg)
+	if err != nil {
+		return err
+	}
+
+	path := getStorePath(cfg)
+
+	return s.Repair(path, cfg)
+}
+
+func init() {
+}

vendor/github.com/lunny/nodb/store/tx.go

@@ -0,0 +1,42 @@
+package store
+
+import (
+	"github.com/lunny/nodb/store/driver"
+)
+
+type Tx struct {
+	driver.Tx
+}
+
+func (tx *Tx) NewIterator() *Iterator {
+	it := new(Iterator)
+	it.it = tx.Tx.NewIterator()
+
+	return it
+}
+
+func (tx *Tx) NewWriteBatch() WriteBatch {
+	return tx.Tx.NewWriteBatch()
+}
+
+func (tx *Tx) RangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator {
+	return NewRangeLimitIterator(tx.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1})
+}
+
+func (tx *Tx) RevRangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator {
+	return NewRevRangeLimitIterator(tx.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1})
+}
+
+//count < 0, unlimit.
+//
+//offset must >= 0, if < 0, will get nothing.
+func (tx *Tx) RangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *RangeLimitIterator {
+	return NewRangeLimitIterator(tx.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count})
+}
+
+//count < 0, unlimit.
+//
+//offset must >= 0, if < 0, will get nothing.
+func (tx *Tx) RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *RangeLimitIterator {
+	return NewRevRangeLimitIterator(tx.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count})
+}

vendor/github.com/lunny/nodb/store/writebatch.go

@@ -0,0 +1,9 @@
+package store
+
+import (
+	"github.com/lunny/nodb/store/driver"
+)
+
+type WriteBatch interface {
+	driver.IWriteBatch
+}

vendor/github.com/lunny/nodb/t_bit.go

@@ -0,0 +1,922 @@
+package nodb
+
+import (
+	"encoding/binary"
+	"errors"
+	"sort"
+	"time"
+
+	"github.com/lunny/nodb/store"
+)
+
+const (
+	OPand uint8 = iota + 1
+	OPor
+	OPxor
+	OPnot
+)
+
+type BitPair struct {
+	Pos int32
+	Val uint8
+}
+
+type segBitInfo struct {
+	Seq uint32
+	Off uint32
+	Val uint8
+}
+
+type segBitInfoArray []segBitInfo
+
+const (
+	// byte
+	segByteWidth uint32 = 9
+	segByteSize  uint32 = 1 << segByteWidth
+
+	// bit
+	segBitWidth uint32 = segByteWidth + 3
+	segBitSize  uint32 = segByteSize << 3
+
+	maxByteSize uint32 = 8 << 20
+	maxSegCount uint32 = maxByteSize / segByteSize
+
+	minSeq uint32 = 0
+	maxSeq uint32 = uint32((maxByteSize << 3) - 1)
+)
+
+var bitsInByte = [256]int32{0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3,
+	4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3,
+	3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4,
+	5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4,
+	3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4,
+	5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2,
+	2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3,
+	4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+	3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4,
+	5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6,
+	6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5,
+	6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}
+
+var fillBits = [...]uint8{1, 3, 7, 15, 31, 63, 127, 255}
+
+var emptySegment []byte = make([]byte, segByteSize, segByteSize)
+
+var fillSegment []byte = func() []byte {
+	data := make([]byte, segByteSize, segByteSize)
+	for i := uint32(0); i < segByteSize; i++ {
+		data[i] = 0xff
+	}
+	return data
+}()
+
+var errBinKey = errors.New("invalid bin key")
+var errOffset = errors.New("invalid offset")
+var errDuplicatePos = errors.New("duplicate bit pos")
+
+func getBit(sz []byte, offset uint32) uint8 {
+	index := offset >> 3
+	if index >= uint32(len(sz)) {
+		return 0 // error("overflow")
+	}
+
+	offset -= index << 3
+	return sz[index] >> offset & 1
+}
+
+func setBit(sz []byte, offset uint32, val uint8) bool {
+	if val != 1 && val != 0 {
+		return false // error("invalid val")
+	}
+
+	index := offset >> 3
+	if index >= uint32(len(sz)) {
+		return false // error("overflow")
+	}
+
+	offset -= index << 3
+	if sz[index]>>offset&1 != val {
+		sz[index] ^= (1 << offset)
+	}
+	return true
+}
+
+func (datas segBitInfoArray) Len() int {
+	return len(datas)
+}
+
+func (datas segBitInfoArray) Less(i, j int) bool {
+	res := (datas)[i].Seq < (datas)[j].Seq
+	if !res && (datas)[i].Seq == (datas)[j].Seq {
+		res = (datas)[i].Off < (datas)[j].Off
+	}
+	return res
+}
+
+func (datas segBitInfoArray) Swap(i, j int) {
+	datas[i], datas[j] = datas[j], datas[i]
+}
+
+func (db *DB) bEncodeMetaKey(key []byte) []byte {
+	mk := make([]byte, len(key)+2)
+	mk[0] = db.index
+	mk[1] = BitMetaType
+
+	copy(mk[2:], key)
+	return mk
+}
+
+func (db *DB) bDecodeMetaKey(bkey []byte) ([]byte, error) {
+	if len(bkey) < 2 || bkey[0] != db.index || bkey[1] != BitMetaType {
+		return nil, errBinKey
+	}
+
+	return bkey[2:], nil
+}
+
+func (db *DB) bEncodeBinKey(key []byte, seq uint32) []byte {
+	bk := make([]byte, len(key)+8)
+
+	pos := 0
+	bk[pos] = db.index
+	pos++
+	bk[pos] = BitType
+	pos++
+
+	binary.BigEndian.PutUint16(bk[pos:], uint16(len(key)))
+	pos += 2
+
+	copy(bk[pos:], key)
+	pos += len(key)
+
+	binary.BigEndian.PutUint32(bk[pos:], seq)
+
+	return bk
+}
+
+func (db *DB) bDecodeBinKey(bkey []byte) (key []byte, seq uint32, err error) {
+	if len(bkey) < 8 || bkey[0] != db.index {
+		err = errBinKey
+		return
+	}
+
+	keyLen := binary.BigEndian.Uint16(bkey[2:4])
+	if int(keyLen+8) != len(bkey) {
+		err = errBinKey
+		return
+	}
+
+	key = bkey[4 : 4+keyLen]
+	seq = uint32(binary.BigEndian.Uint32(bkey[4+keyLen:]))
+	return
+}
+
+func (db *DB) bCapByteSize(seq uint32, off uint32) uint32 {
+	var offByteSize uint32 = (off >> 3) + 1
+	if offByteSize > segByteSize {
+		offByteSize = segByteSize
+	}
+
+	return seq<<segByteWidth + offByteSize
+}
+
+func (db *DB) bParseOffset(key []byte, offset int32) (seq uint32, off uint32, err error) {
+	if offset < 0 {
+		if tailSeq, tailOff, e := db.bGetMeta(key); e != nil {
+			err = e
+			return
+		} else if tailSeq >= 0 {
+			offset += int32((uint32(tailSeq)<<segBitWidth | uint32(tailOff)) + 1)
+			if offset < 0 {
+				err = errOffset
+				return
+			}
+		}
+	}
+
+	off = uint32(offset)
+
+	seq = off >> segBitWidth
+	off &= (segBitSize - 1)
+	return
+}
+
+func (db *DB) bGetMeta(key []byte) (tailSeq int32, tailOff int32, err error) {
+	var v []byte
+
+	mk := db.bEncodeMetaKey(key)
+	v, err = db.bucket.Get(mk)
+	if err != nil {
+		return
+	}
+
+	if v != nil {
+		tailSeq = int32(binary.LittleEndian.Uint32(v[0:4]))
+		tailOff = int32(binary.LittleEndian.Uint32(v[4:8]))
+	} else {
+		tailSeq = -1
+		tailOff = -1
+	}
+	return
+}
+
+func (db *DB) bSetMeta(t *batch, key []byte, tailSeq uint32, tailOff uint32) {
+	ek := db.bEncodeMetaKey(key)
+
+	buf := make([]byte, 8)
+	binary.LittleEndian.PutUint32(buf[0:4], tailSeq)
+	binary.LittleEndian.PutUint32(buf[4:8], tailOff)
+
+	t.Put(ek, buf)
+	return
+}
+
+func (db *DB) bUpdateMeta(t *batch, key []byte, seq uint32, off uint32) (tailSeq uint32, tailOff uint32, err error) {
+	var tseq, toff int32
+	var update bool = false
+
+	if tseq, toff, err = db.bGetMeta(key); err != nil {
+		return
+	} else if tseq < 0 {
+		update = true
+	} else {
+		tailSeq = uint32(MaxInt32(tseq, 0))
+		tailOff = uint32(MaxInt32(toff, 0))
+		update = (seq > tailSeq || (seq == tailSeq && off > tailOff))
+	}
+
+	if update {
+		db.bSetMeta(t, key, seq, off)
+		tailSeq = seq
+		tailOff = off
+	}
+	return
+}
+
+func (db *DB) bDelete(t *batch, key []byte) (drop int64) {
+	mk := db.bEncodeMetaKey(key)
+	t.Delete(mk)
+
+	minKey := db.bEncodeBinKey(key, minSeq)
+	maxKey := db.bEncodeBinKey(key, maxSeq)
+	it := db.bucket.RangeIterator(minKey, maxKey, store.RangeClose)
+	for ; it.Valid(); it.Next() {
+		t.Delete(it.RawKey())
+		drop++
+	}
+	it.Close()
+
+	return drop
+}
+
+func (db *DB) bGetSegment(key []byte, seq uint32) ([]byte, []byte, error) {
+	bk := db.bEncodeBinKey(key, seq)
+	segment, err := db.bucket.Get(bk)
+	if err != nil {
+		return bk, nil, err
+	}
+	return bk, segment, nil
+}
+
+func (db *DB) bAllocateSegment(key []byte, seq uint32) ([]byte, []byte, error) {
+	bk, segment, err := db.bGetSegment(key, seq)
+	if err == nil && segment == nil {
+		segment = make([]byte, segByteSize, segByteSize)
+	}
+	return bk, segment, err
+}
+
+func (db *DB) bIterator(key []byte) *store.RangeLimitIterator {
+	sk := db.bEncodeBinKey(key, minSeq)
+	ek := db.bEncodeBinKey(key, maxSeq)
+	return db.bucket.RangeIterator(sk, ek, store.RangeClose)
+}
+
+func (db *DB) bSegAnd(a []byte, b []byte, res *[]byte) {
+	if a == nil || b == nil {
+		*res = nil
+		return
+	}
+
+	data := *res
+	if data == nil {
+		data = make([]byte, segByteSize, segByteSize)
+		*res = data
+	}
+
+	for i := uint32(0); i < segByteSize; i++ {
+		data[i] = a[i] & b[i]
+	}
+	return
+}
+
+func (db *DB) bSegOr(a []byte, b []byte, res *[]byte) {
+	if a == nil || b == nil {
+		if a == nil && b == nil {
+			*res = nil
+		} else if a == nil {
+			*res = b
+		} else {
+			*res = a
+		}
+		return
+	}
+
+	data := *res
+	if data == nil {
+		data = make([]byte, segByteSize, segByteSize)
+		*res = data
+	}
+
+	for i := uint32(0); i < segByteSize; i++ {
+		data[i] = a[i] | b[i]
+	}
+	return
+}
+
+func (db *DB) bSegXor(a []byte, b []byte, res *[]byte) {
+	if a == nil && b == nil {
+		*res = fillSegment
+		return
+	}
+
+	if a == nil {
+		a = emptySegment
+	}
+
+	if b == nil {
+		b = emptySegment
+	}
+
+	data := *res
+	if data == nil {
+		data = make([]byte, segByteSize, segByteSize)
+		*res = data
+	}
+
+	for i := uint32(0); i < segByteSize; i++ {
+		data[i] = a[i] ^ b[i]
+	}
+
+	return
+}
+
+func (db *DB) bExpireAt(key []byte, when int64) (int64, error) {
+	t := db.binBatch
+	t.Lock()
+	defer t.Unlock()
+
+	if seq, _, err := db.bGetMeta(key); err != nil || seq < 0 {
+		return 0, err
+	} else {
+		db.expireAt(t, BitType, key, when)
+		if err := t.Commit(); err != nil {
+			return 0, err
+		}
+	}
+	return 1, nil
+}
+
+func (db *DB) bCountByte(val byte, soff uint32, eoff uint32) int32 {
+	if soff > eoff {
+		soff, eoff = eoff, soff
+	}
+
+	mask := uint8(0)
+	if soff > 0 {
+		mask |= fillBits[soff-1]
+	}
+	if eoff < 7 {
+		mask |= (fillBits[7] ^ fillBits[eoff])
+	}
+	mask = fillBits[7] ^ mask
+
+	return bitsInByte[val&mask]
+}
+
+func (db *DB) bCountSeg(key []byte, seq uint32, soff uint32, eoff uint32) (cnt int32, err error) {
+	if soff >= segBitSize || soff < 0 ||
+		eoff >= segBitSize || eoff < 0 {
+		return
+	}
+
+	var segment []byte
+	if _, segment, err = db.bGetSegment(key, seq); err != nil {
+		return
+	}
+
+	if segment == nil {
+		return
+	}
+
+	if soff > eoff {
+		soff, eoff = eoff, soff
+	}
+
+	headIdx := int(soff >> 3)
+	endIdx := int(eoff >> 3)
+	sByteOff := soff - ((soff >> 3) << 3)
+	eByteOff := eoff - ((eoff >> 3) << 3)
+
+	if headIdx == endIdx {
+		cnt = db.bCountByte(segment[headIdx], sByteOff, eByteOff)
+	} else {
+		cnt = db.bCountByte(segment[headIdx], sByteOff, 7) +
+			db.bCountByte(segment[endIdx], 0, eByteOff)
+	}
+
+	// sum up following bytes
+	for idx, end := headIdx+1, endIdx-1; idx <= end; idx += 1 {
+		cnt += bitsInByte[segment[idx]]
+		if idx == end {
+			break
+		}
+	}
+
+	return
+}
+
+func (db *DB) BGet(key []byte) (data []byte, err error) {
+	if err = checkKeySize(key); err != nil {
+		return
+	}
+
+	var ts, to int32
+	if ts, to, err = db.bGetMeta(key); err != nil || ts < 0 {
+		return
+	}
+
+	var tailSeq, tailOff = uint32(ts), uint32(to)
+	var capByteSize uint32 = db.bCapByteSize(tailSeq, tailOff)
+	data = make([]byte, capByteSize, capByteSize)
+
+	minKey := db.bEncodeBinKey(key, minSeq)
+	maxKey := db.bEncodeBinKey(key, tailSeq)
+	it := db.bucket.RangeIterator(minKey, maxKey, store.RangeClose)
+
+	var seq, s, e uint32
+	for ; it.Valid(); it.Next() {
+		if _, seq, err = db.bDecodeBinKey(it.RawKey()); err != nil {
+			data = nil
+			break
+		}
+
+		s = seq << segByteWidth
+		e = MinUInt32(s+segByteSize, capByteSize)
+		copy(data[s:e], it.RawValue())
+	}
+	it.Close()
+
+	return
+}
+
+func (db *DB) BDelete(key []byte) (drop int64, err error) {
+	if err = checkKeySize(key); err != nil {
+		return
+	}
+
+	t := db.binBatch
+	t.Lock()
+	defer t.Unlock()
+
+	drop = db.bDelete(t, key)
+	db.rmExpire(t, BitType, key)
+
+	err = t.Commit()
+	return
+}
+
+func (db *DB) BSetBit(key []byte, offset int32, val uint8) (ori uint8, err error) {
+	if err = checkKeySize(key); err != nil {
+		return
+	}
+
+	//	todo : check offset
+	var seq, off uint32
+	if seq, off, err = db.bParseOffset(key, offset); err != nil {
+		return 0, err
+	}
+
+	var bk, segment []byte
+	if bk, segment, err = db.bAllocateSegment(key, seq); err != nil {
+		return 0, err
+	}
+
+	if segment != nil {
+		ori = getBit(segment, off)
+		if setBit(segment, off, val) {
+			t := db.binBatch
+			t.Lock()
+			defer t.Unlock()
+
+			t.Put(bk, segment)
+			if _, _, e := db.bUpdateMeta(t, key, seq, off); e != nil {
+				err = e
+				return
+			}
+
+			err = t.Commit()
+		}
+	}
+
+	return
+}
+
+func (db *DB) BMSetBit(key []byte, args ...BitPair) (place int64, err error) {
+	if err = checkKeySize(key); err != nil {
+		return
+	}
+
+	//	(ps : so as to aviod wasting memory copy while calling db.Get() and batch.Put(),
+	//		  here we sequence the params by pos, so that we can merge the execution of
+	//		  diff pos setting which targets on the same segment respectively. )
+
+	//	#1 : sequence request data
+	var argCnt = len(args)
+	var bitInfos segBitInfoArray = make(segBitInfoArray, argCnt)
+	var seq, off uint32
+
+	for i, info := range args {
+		if seq, off, err = db.bParseOffset(key, info.Pos); err != nil {
+			return
+		}
+
+		bitInfos[i].Seq = seq
+		bitInfos[i].Off = off
+		bitInfos[i].Val = info.Val
+	}
+
+	sort.Sort(bitInfos)
+
+	for i := 1; i < argCnt; i++ {
+		if bitInfos[i].Seq == bitInfos[i-1].Seq && bitInfos[i].Off == bitInfos[i-1].Off {
+			return 0, errDuplicatePos
+		}
+	}
+
+	//	#2 : execute bit set in order
+	t := db.binBatch
+	t.Lock()
+	defer t.Unlock()
+
+	var curBinKey, curSeg []byte
+	var curSeq, maxSeq, maxOff uint32
+
+	for _, info := range bitInfos {
+		if curSeg != nil && info.Seq != curSeq {
+			t.Put(curBinKey, curSeg)
+			curSeg = nil
+		}
+
+		if curSeg == nil {
+			curSeq = info.Seq
+			if curBinKey, curSeg, err = db.bAllocateSegment(key, info.Seq); err != nil {
+				return
+			}
+
+			if curSeg == nil {
+				continue
+			}
+		}
+
+		if setBit(curSeg, info.Off, info.Val) {
+			maxSeq = info.Seq
+			maxOff = info.Off
+			place++
+		}
+	}
+
+	if curSeg != nil {
+		t.Put(curBinKey, curSeg)
+	}
+
+	//	finally, update meta
+	if place > 0 {
+		if _, _, err = db.bUpdateMeta(t, key, maxSeq, maxOff); err != nil {
+			return
+		}
+
+		err = t.Commit()
+	}
+
+	return
+}
+
+func (db *DB) BGetBit(key []byte, offset int32) (uint8, error) {
+	if seq, off, err := db.bParseOffset(key, offset); err != nil {
+		return 0, err
+	} else {
+		_, segment, err := db.bGetSegment(key, seq)
+		if err != nil {
+			return 0, err
+		}
+
+		if segment == nil {
+			return 0, nil
+		} else {
+			return getBit(segment, off), nil
+		}
+	}
+}
+
+// func (db *DB) BGetRange(key []byte, start int32, end int32) ([]byte, error) {
+// 	section := make([]byte)
+
+// 	return
+// }
+
+func (db *DB) BCount(key []byte, start int32, end int32) (cnt int32, err error) {
+	var sseq, soff uint32
+	if sseq, soff, err = db.bParseOffset(key, start); err != nil {
+		return
+	}
+
+	var eseq, eoff uint32
+	if eseq, eoff, err = db.bParseOffset(key, end); err != nil {
+		return
+	}
+
+	if sseq > eseq || (sseq == eseq && soff > eoff) {
+		sseq, eseq = eseq, sseq
+		soff, eoff = eoff, soff
+	}
+
+	var segCnt int32
+	if eseq == sseq {
+		if segCnt, err = db.bCountSeg(key, sseq, soff, eoff); err != nil {
+			return 0, err
+		}
+
+		cnt = segCnt
+
+	} else {
+		if segCnt, err = db.bCountSeg(key, sseq, soff, segBitSize-1); err != nil {
+			return 0, err
+		} else {
+			cnt += segCnt
+		}
+
+		if segCnt, err = db.bCountSeg(key, eseq, 0, eoff); err != nil {
+			return 0, err
+		} else {
+			cnt += segCnt
+		}
+	}
+
+	//	middle segs
+	var segment []byte
+	skey := db.bEncodeBinKey(key, sseq)
+	ekey := db.bEncodeBinKey(key, eseq)
+
+	it := db.bucket.RangeIterator(skey, ekey, store.RangeOpen)
+	for ; it.Valid(); it.Next() {
+		segment = it.RawValue()
+		for _, bt := range segment {
+			cnt += bitsInByte[bt]
+		}
+	}
+	it.Close()
+
+	return
+}
+
+func (db *DB) BTail(key []byte) (int32, error) {
+	// effective length of data, the highest bit-pos set in history
+	tailSeq, tailOff, err := db.bGetMeta(key)
+	if err != nil {
+		return 0, err
+	}
+
+	tail := int32(-1)
+	if tailSeq >= 0 {
+		tail = int32(uint32(tailSeq)<<segBitWidth | uint32(tailOff))
+	}
+
+	return tail, nil
+}
+
+func (db *DB) BOperation(op uint8, dstkey []byte, srckeys ...[]byte) (blen int32, err error) {
+	//	blen -
+	//		the total bit size of data stored in destination key,
+	//		that is equal to the size of the longest input string.
+
+	var exeOp func([]byte, []byte, *[]byte)
+	switch op {
+	case OPand:
+		exeOp = db.bSegAnd
+	case OPor:
+		exeOp = db.bSegOr
+	case OPxor, OPnot:
+		exeOp = db.bSegXor
+	default:
+		return
+	}
+
+	if dstkey == nil || srckeys == nil {
+		return
+	}
+
+	t := db.binBatch
+	t.Lock()
+	defer t.Unlock()
+
+	var srcKseq, srcKoff int32
+	var seq, off, maxDstSeq, maxDstOff uint32
+
+	var keyNum int = len(srckeys)
+	var validKeyNum int
+	for i := 0; i < keyNum; i++ {
+		if srcKseq, srcKoff, err = db.bGetMeta(srckeys[i]); err != nil {
+			return
+		} else if srcKseq < 0 {
+			srckeys[i] = nil
+			continue
+		}
+
+		validKeyNum++
+
+		seq = uint32(srcKseq)
+		off = uint32(srcKoff)
+		if seq > maxDstSeq || (seq == maxDstSeq && off > maxDstOff) {
+			maxDstSeq = seq
+			maxDstOff = off
+		}
+	}
+
+	if (op == OPnot && validKeyNum != 1) ||
+		(op != OPnot && validKeyNum < 2) {
+		return // with not enough existing source key
+	}
+
+	var srcIdx int
+	for srcIdx = 0; srcIdx < keyNum; srcIdx++ {
+		if srckeys[srcIdx] != nil {
+			break
+		}
+	}
+
+	// init - data
+	var segments = make([][]byte, maxDstSeq+1)
+
+	if op == OPnot {
+		//	ps :
+		//		( ~num == num ^ 0x11111111 )
+		//		we init the result segments with all bit set,
+		//		then we can calculate through the way of 'xor'.
+
+		//	ahead segments bin format : 1111 ... 1111
+		for i := uint32(0); i < maxDstSeq; i++ {
+			segments[i] = fillSegment
+		}
+
+		//	last segment bin format : 1111..1100..0000
+		var tailSeg = make([]byte, segByteSize, segByteSize)
+		var fillByte = fillBits[7]
+		var tailSegLen = db.bCapByteSize(uint32(0), maxDstOff)
+		for i := uint32(0); i < tailSegLen-1; i++ {
+			tailSeg[i] = fillByte
+		}
+		tailSeg[tailSegLen-1] = fillBits[maxDstOff-(tailSegLen-1)<<3]
+		segments[maxDstSeq] = tailSeg
+
+	} else {
+		// ps : init segments by data corresponding to the 1st valid source key
+		it := db.bIterator(srckeys[srcIdx])
+		for ; it.Valid(); it.Next() {
+			if _, seq, err = db.bDecodeBinKey(it.RawKey()); err != nil {
+				// to do ...
+				it.Close()
+				return
+			}
+			segments[seq] = it.Value()
+		}
+		it.Close()
+		srcIdx++
+	}
+
+	//	operation with following keys
+	var res []byte
+	for i := srcIdx; i < keyNum; i++ {
+		if srckeys[i] == nil {
+			continue
+		}
+
+		it := db.bIterator(srckeys[i])
+		for idx, end := uint32(0), false; !end; it.Next() {
+			end = !it.Valid()
+			if !end {
+				if _, seq, err = db.bDecodeBinKey(it.RawKey()); err != nil {
+					// to do ...
+					it.Close()
+					return
+				}
+			} else {
+				seq = maxDstSeq + 1
+			}
+
+			// todo :
+			// 		operation 'and' can be optimize here :
+			//		if seq > max_segments_idx, this loop can be break,
+			//		which can avoid cost from Key() and bDecodeBinKey()
+
+			for ; idx < seq; idx++ {
+				res = nil
+				exeOp(segments[idx], nil, &res)
+				segments[idx] = res
+			}
+
+			if !end {
+				res = it.Value()
+				exeOp(segments[seq], res, &res)
+				segments[seq] = res
+				idx++
+			}
+		}
+		it.Close()
+	}
+
+	// clear the old data in case
+	db.bDelete(t, dstkey)
+	db.rmExpire(t, BitType, dstkey)
+
+	//	set data
+	db.bSetMeta(t, dstkey, maxDstSeq, maxDstOff)
+
+	var bk []byte
+	for seq, segt := range segments {
+		if segt != nil {
+			bk = db.bEncodeBinKey(dstkey, uint32(seq))
+			t.Put(bk, segt)
+		}
+	}
+
+	err = t.Commit()
+	if err == nil {
+		// blen = int32(db.bCapByteSize(maxDstOff, maxDstOff))
+		blen = int32(maxDstSeq<<segBitWidth | maxDstOff + 1)
+	}
+
+	return
+}
+
+func (db *DB) BExpire(key []byte, duration int64) (int64, error) {
+	if duration <= 0 {
+		return 0, errExpireValue
+	}
+
+	if err := checkKeySize(key); err != nil {
+		return -1, err
+	}
+
+	return db.bExpireAt(key, time.Now().Unix()+duration)
+}
+
+func (db *DB) BExpireAt(key []byte, when int64) (int64, error) {
+	if when <= time.Now().Unix() {
+		return 0, errExpireValue
+	}
+
+	if err := checkKeySize(key); err != nil {
+		return -1, err
+	}
+
+	return db.bExpireAt(key, when)
+}
+
+func (db *DB) BTTL(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return -1, err
+	}
+
+	return db.ttl(BitType, key)
+}
+
+func (db *DB) BPersist(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.binBatch
+	t.Lock()
+	defer t.Unlock()
+
+	n, err := db.rmExpire(t, BitType, key)
+	if err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+	return n, err
+}
+
+func (db *DB) BScan(key []byte, count int, inclusive bool, match string) ([][]byte, error) {
+	return db.scan(BitMetaType, key, count, inclusive, match)
+}
+
+func (db *DB) bFlush() (drop int64, err error) {
+	t := db.binBatch
+	t.Lock()
+	defer t.Unlock()
+
+	return db.flushType(t, BitType)
+}

vendor/github.com/lunny/nodb/t_hash.go

@@ -0,0 +1,509 @@
+package nodb
+
+import (
+	"encoding/binary"
+	"errors"
+	"time"
+
+	"github.com/lunny/nodb/store"
+)
+
+type FVPair struct {
+	Field []byte
+	Value []byte
+}
+
+var errHashKey = errors.New("invalid hash key")
+var errHSizeKey = errors.New("invalid hsize key")
+
+const (
+	hashStartSep byte = ':'
+	hashStopSep  byte = hashStartSep + 1
+)
+
+func checkHashKFSize(key []byte, field []byte) error {
+	if len(key) > MaxKeySize || len(key) == 0 {
+		return errKeySize
+	} else if len(field) > MaxHashFieldSize || len(field) == 0 {
+		return errHashFieldSize
+	}
+	return nil
+}
+
+func (db *DB) hEncodeSizeKey(key []byte) []byte {
+	buf := make([]byte, len(key)+2)
+
+	buf[0] = db.index
+	buf[1] = HSizeType
+
+	copy(buf[2:], key)
+	return buf
+}
+
+func (db *DB) hDecodeSizeKey(ek []byte) ([]byte, error) {
+	if len(ek) < 2 || ek[0] != db.index || ek[1] != HSizeType {
+		return nil, errHSizeKey
+	}
+
+	return ek[2:], nil
+}
+
+func (db *DB) hEncodeHashKey(key []byte, field []byte) []byte {
+	buf := make([]byte, len(key)+len(field)+1+1+2+1)
+
+	pos := 0
+	buf[pos] = db.index
+	pos++
+	buf[pos] = HashType
+	pos++
+
+	binary.BigEndian.PutUint16(buf[pos:], uint16(len(key)))
+	pos += 2
+
+	copy(buf[pos:], key)
+	pos += len(key)
+
+	buf[pos] = hashStartSep
+	pos++
+	copy(buf[pos:], field)
+
+	return buf
+}
+
+func (db *DB) hDecodeHashKey(ek []byte) ([]byte, []byte, error) {
+	if len(ek) < 5 || ek[0] != db.index || ek[1] != HashType {
+		return nil, nil, errHashKey
+	}
+
+	pos := 2
+	keyLen := int(binary.BigEndian.Uint16(ek[pos:]))
+	pos += 2
+
+	if keyLen+5 > len(ek) {
+		return nil, nil, errHashKey
+	}
+
+	key := ek[pos : pos+keyLen]
+	pos += keyLen
+
+	if ek[pos] != hashStartSep {
+		return nil, nil, errHashKey
+	}
+
+	pos++
+	field := ek[pos:]
+	return key, field, nil
+}
+
+func (db *DB) hEncodeStartKey(key []byte) []byte {
+	return db.hEncodeHashKey(key, nil)
+}
+
+func (db *DB) hEncodeStopKey(key []byte) []byte {
+	k := db.hEncodeHashKey(key, nil)
+
+	k[len(k)-1] = hashStopSep
+
+	return k
+}
+
+func (db *DB) hSetItem(key []byte, field []byte, value []byte) (int64, error) {
+	t := db.hashBatch
+
+	ek := db.hEncodeHashKey(key, field)
+
+	var n int64 = 1
+	if v, _ := db.bucket.Get(ek); v != nil {
+		n = 0
+	} else {
+		if _, err := db.hIncrSize(key, 1); err != nil {
+			return 0, err
+		}
+	}
+
+	t.Put(ek, value)
+	return n, nil
+}
+
+//	ps : here just focus on deleting the hash data,
+//		 any other likes expire is ignore.
+func (db *DB) hDelete(t *batch, key []byte) int64 {
+	sk := db.hEncodeSizeKey(key)
+	start := db.hEncodeStartKey(key)
+	stop := db.hEncodeStopKey(key)
+
+	var num int64 = 0
+	it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1)
+	for ; it.Valid(); it.Next() {
+		t.Delete(it.Key())
+		num++
+	}
+	it.Close()
+
+	t.Delete(sk)
+	return num
+}
+
+func (db *DB) hExpireAt(key []byte, when int64) (int64, error) {
+	t := db.hashBatch
+	t.Lock()
+	defer t.Unlock()
+
+	if hlen, err := db.HLen(key); err != nil || hlen == 0 {
+		return 0, err
+	} else {
+		db.expireAt(t, HashType, key, when)
+		if err := t.Commit(); err != nil {
+			return 0, err
+		}
+	}
+	return 1, nil
+}
+
+func (db *DB) HLen(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	return Int64(db.bucket.Get(db.hEncodeSizeKey(key)))
+}
+
+func (db *DB) HSet(key []byte, field []byte, value []byte) (int64, error) {
+	if err := checkHashKFSize(key, field); err != nil {
+		return 0, err
+	} else if err := checkValueSize(value); err != nil {
+		return 0, err
+	}
+
+	t := db.hashBatch
+	t.Lock()
+	defer t.Unlock()
+
+	n, err := db.hSetItem(key, field, value)
+	if err != nil {
+		return 0, err
+	}
+
+	//todo add binlog
+
+	err = t.Commit()
+	return n, err
+}
+
+func (db *DB) HGet(key []byte, field []byte) ([]byte, error) {
+	if err := checkHashKFSize(key, field); err != nil {
+		return nil, err
+	}
+
+	return db.bucket.Get(db.hEncodeHashKey(key, field))
+}
+
+func (db *DB) HMset(key []byte, args ...FVPair) error {
+	t := db.hashBatch
+	t.Lock()
+	defer t.Unlock()
+
+	var err error
+	var ek []byte
+	var num int64 = 0
+	for i := 0; i < len(args); i++ {
+		if err := checkHashKFSize(key, args[i].Field); err != nil {
+			return err
+		} else if err := checkValueSize(args[i].Value); err != nil {
+			return err
+		}
+
+		ek = db.hEncodeHashKey(key, args[i].Field)
+
+		if v, err := db.bucket.Get(ek); err != nil {
+			return err
+		} else if v == nil {
+			num++
+		}
+
+		t.Put(ek, args[i].Value)
+	}
+
+	if _, err = db.hIncrSize(key, num); err != nil {
+		return err
+	}
+
+	//todo add binglog
+	err = t.Commit()
+	return err
+}
+
+func (db *DB) HMget(key []byte, args ...[]byte) ([][]byte, error) {
+	var ek []byte
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	r := make([][]byte, len(args))
+	for i := 0; i < len(args); i++ {
+		if err := checkHashKFSize(key, args[i]); err != nil {
+			return nil, err
+		}
+
+		ek = db.hEncodeHashKey(key, args[i])
+
+		r[i] = it.Find(ek)
+	}
+
+	return r, nil
+}
+
+func (db *DB) HDel(key []byte, args ...[]byte) (int64, error) {
+	t := db.hashBatch
+
+	var ek []byte
+	var v []byte
+	var err error
+
+	t.Lock()
+	defer t.Unlock()
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	var num int64 = 0
+	for i := 0; i < len(args); i++ {
+		if err := checkHashKFSize(key, args[i]); err != nil {
+			return 0, err
+		}
+
+		ek = db.hEncodeHashKey(key, args[i])
+
+		v = it.RawFind(ek)
+		if v == nil {
+			continue
+		} else {
+			num++
+			t.Delete(ek)
+		}
+	}
+
+	if _, err = db.hIncrSize(key, -num); err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+
+	return num, err
+}
+
+func (db *DB) hIncrSize(key []byte, delta int64) (int64, error) {
+	t := db.hashBatch
+	sk := db.hEncodeSizeKey(key)
+
+	var err error
+	var size int64 = 0
+	if size, err = Int64(db.bucket.Get(sk)); err != nil {
+		return 0, err
+	} else {
+		size += delta
+		if size <= 0 {
+			size = 0
+			t.Delete(sk)
+			db.rmExpire(t, HashType, key)
+		} else {
+			t.Put(sk, PutInt64(size))
+		}
+	}
+
+	return size, nil
+}
+
+func (db *DB) HIncrBy(key []byte, field []byte, delta int64) (int64, error) {
+	if err := checkHashKFSize(key, field); err != nil {
+		return 0, err
+	}
+
+	t := db.hashBatch
+	var ek []byte
+	var err error
+
+	t.Lock()
+	defer t.Unlock()
+
+	ek = db.hEncodeHashKey(key, field)
+
+	var n int64 = 0
+	if n, err = StrInt64(db.bucket.Get(ek)); err != nil {
+		return 0, err
+	}
+
+	n += delta
+
+	_, err = db.hSetItem(key, field, StrPutInt64(n))
+	if err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+
+	return n, err
+}
+
+func (db *DB) HGetAll(key []byte) ([]FVPair, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	start := db.hEncodeStartKey(key)
+	stop := db.hEncodeStopKey(key)
+
+	v := make([]FVPair, 0, 16)
+
+	it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1)
+	for ; it.Valid(); it.Next() {
+		_, f, err := db.hDecodeHashKey(it.Key())
+		if err != nil {
+			return nil, err
+		}
+
+		v = append(v, FVPair{Field: f, Value: it.Value()})
+	}
+
+	it.Close()
+
+	return v, nil
+}
+
+func (db *DB) HKeys(key []byte) ([][]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	start := db.hEncodeStartKey(key)
+	stop := db.hEncodeStopKey(key)
+
+	v := make([][]byte, 0, 16)
+
+	it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1)
+	for ; it.Valid(); it.Next() {
+		_, f, err := db.hDecodeHashKey(it.Key())
+		if err != nil {
+			return nil, err
+		}
+		v = append(v, f)
+	}
+
+	it.Close()
+
+	return v, nil
+}
+
+func (db *DB) HValues(key []byte) ([][]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	start := db.hEncodeStartKey(key)
+	stop := db.hEncodeStopKey(key)
+
+	v := make([][]byte, 0, 16)
+
+	it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1)
+	for ; it.Valid(); it.Next() {
+		_, _, err := db.hDecodeHashKey(it.Key())
+		if err != nil {
+			return nil, err
+		}
+
+		v = append(v, it.Value())
+	}
+
+	it.Close()
+
+	return v, nil
+}
+
+func (db *DB) HClear(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.hashBatch
+	t.Lock()
+	defer t.Unlock()
+
+	num := db.hDelete(t, key)
+	db.rmExpire(t, HashType, key)
+
+	err := t.Commit()
+	return num, err
+}
+
+func (db *DB) HMclear(keys ...[]byte) (int64, error) {
+	t := db.hashBatch
+	t.Lock()
+	defer t.Unlock()
+
+	for _, key := range keys {
+		if err := checkKeySize(key); err != nil {
+			return 0, err
+		}
+
+		db.hDelete(t, key)
+		db.rmExpire(t, HashType, key)
+	}
+
+	err := t.Commit()
+	return int64(len(keys)), err
+}
+
+func (db *DB) hFlush() (drop int64, err error) {
+	t := db.hashBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	return db.flushType(t, HashType)
+}
+
+func (db *DB) HScan(key []byte, count int, inclusive bool, match string) ([][]byte, error) {
+	return db.scan(HSizeType, key, count, inclusive, match)
+}
+
+func (db *DB) HExpire(key []byte, duration int64) (int64, error) {
+	if duration <= 0 {
+		return 0, errExpireValue
+	}
+
+	return db.hExpireAt(key, time.Now().Unix()+duration)
+}
+
+func (db *DB) HExpireAt(key []byte, when int64) (int64, error) {
+	if when <= time.Now().Unix() {
+		return 0, errExpireValue
+	}
+
+	return db.hExpireAt(key, when)
+}
+
+func (db *DB) HTTL(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return -1, err
+	}
+
+	return db.ttl(HashType, key)
+}
+
+func (db *DB) HPersist(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.hashBatch
+	t.Lock()
+	defer t.Unlock()
+
+	n, err := db.rmExpire(t, HashType, key)
+	if err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+	return n, err
+}

vendor/github.com/lunny/nodb/t_kv.go

@@ -0,0 +1,387 @@
+package nodb
+
+import (
+	"errors"
+	"time"
+)
+
+type KVPair struct {
+	Key   []byte
+	Value []byte
+}
+
+var errKVKey = errors.New("invalid encode kv key")
+
+func checkKeySize(key []byte) error {
+	if len(key) > MaxKeySize || len(key) == 0 {
+		return errKeySize
+	}
+	return nil
+}
+
+func checkValueSize(value []byte) error {
+	if len(value) > MaxValueSize {
+		return errValueSize
+	}
+
+	return nil
+}
+
+func (db *DB) encodeKVKey(key []byte) []byte {
+	ek := make([]byte, len(key)+2)
+	ek[0] = db.index
+	ek[1] = KVType
+	copy(ek[2:], key)
+	return ek
+}
+
+func (db *DB) decodeKVKey(ek []byte) ([]byte, error) {
+	if len(ek) < 2 || ek[0] != db.index || ek[1] != KVType {
+		return nil, errKVKey
+	}
+
+	return ek[2:], nil
+}
+
+func (db *DB) encodeKVMinKey() []byte {
+	ek := db.encodeKVKey(nil)
+	return ek
+}
+
+func (db *DB) encodeKVMaxKey() []byte {
+	ek := db.encodeKVKey(nil)
+	ek[len(ek)-1] = KVType + 1
+	return ek
+}
+
+func (db *DB) incr(key []byte, delta int64) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	var err error
+	key = db.encodeKVKey(key)
+
+	t := db.kvBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	var n int64
+	n, err = StrInt64(db.bucket.Get(key))
+	if err != nil {
+		return 0, err
+	}
+
+	n += delta
+
+	t.Put(key, StrPutInt64(n))
+
+	//todo binlog
+
+	err = t.Commit()
+	return n, err
+}
+
+//	ps : here just focus on deleting the key-value data,
+//		 any other likes expire is ignore.
+func (db *DB) delete(t *batch, key []byte) int64 {
+	key = db.encodeKVKey(key)
+	t.Delete(key)
+	return 1
+}
+
+func (db *DB) setExpireAt(key []byte, when int64) (int64, error) {
+	t := db.kvBatch
+	t.Lock()
+	defer t.Unlock()
+
+	if exist, err := db.Exists(key); err != nil || exist == 0 {
+		return 0, err
+	} else {
+		db.expireAt(t, KVType, key, when)
+		if err := t.Commit(); err != nil {
+			return 0, err
+		}
+	}
+	return 1, nil
+}
+
+func (db *DB) Decr(key []byte) (int64, error) {
+	return db.incr(key, -1)
+}
+
+func (db *DB) DecrBy(key []byte, decrement int64) (int64, error) {
+	return db.incr(key, -decrement)
+}
+
+func (db *DB) Del(keys ...[]byte) (int64, error) {
+	if len(keys) == 0 {
+		return 0, nil
+	}
+
+	codedKeys := make([][]byte, len(keys))
+	for i, k := range keys {
+		codedKeys[i] = db.encodeKVKey(k)
+	}
+
+	t := db.kvBatch
+	t.Lock()
+	defer t.Unlock()
+
+	for i, k := range keys {
+		t.Delete(codedKeys[i])
+		db.rmExpire(t, KVType, k)
+	}
+
+	err := t.Commit()
+	return int64(len(keys)), err
+}
+
+func (db *DB) Exists(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	var err error
+	key = db.encodeKVKey(key)
+
+	var v []byte
+	v, err = db.bucket.Get(key)
+	if v != nil && err == nil {
+		return 1, nil
+	}
+
+	return 0, err
+}
+
+func (db *DB) Get(key []byte) ([]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	key = db.encodeKVKey(key)
+
+	return db.bucket.Get(key)
+}
+
+func (db *DB) GetSet(key []byte, value []byte) ([]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	} else if err := checkValueSize(value); err != nil {
+		return nil, err
+	}
+
+	key = db.encodeKVKey(key)
+
+	t := db.kvBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	oldValue, err := db.bucket.Get(key)
+	if err != nil {
+		return nil, err
+	}
+
+	t.Put(key, value)
+	//todo, binlog
+
+	err = t.Commit()
+
+	return oldValue, err
+}
+
+func (db *DB) Incr(key []byte) (int64, error) {
+	return db.incr(key, 1)
+}
+
+func (db *DB) IncrBy(key []byte, increment int64) (int64, error) {
+	return db.incr(key, increment)
+}
+
+func (db *DB) MGet(keys ...[]byte) ([][]byte, error) {
+	values := make([][]byte, len(keys))
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	for i := range keys {
+		if err := checkKeySize(keys[i]); err != nil {
+			return nil, err
+		}
+
+		values[i] = it.Find(db.encodeKVKey(keys[i]))
+	}
+
+	return values, nil
+}
+
+func (db *DB) MSet(args ...KVPair) error {
+	if len(args) == 0 {
+		return nil
+	}
+
+	t := db.kvBatch
+
+	var err error
+	var key []byte
+	var value []byte
+
+	t.Lock()
+	defer t.Unlock()
+
+	for i := 0; i < len(args); i++ {
+		if err := checkKeySize(args[i].Key); err != nil {
+			return err
+		} else if err := checkValueSize(args[i].Value); err != nil {
+			return err
+		}
+
+		key = db.encodeKVKey(args[i].Key)
+
+		value = args[i].Value
+
+		t.Put(key, value)
+
+		//todo binlog
+	}
+
+	err = t.Commit()
+	return err
+}
+
+func (db *DB) Set(key []byte, value []byte) error {
+	if err := checkKeySize(key); err != nil {
+		return err
+	} else if err := checkValueSize(value); err != nil {
+		return err
+	}
+
+	var err error
+	key = db.encodeKVKey(key)
+
+	t := db.kvBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	t.Put(key, value)
+
+	err = t.Commit()
+
+	return err
+}
+
+func (db *DB) SetNX(key []byte, value []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	} else if err := checkValueSize(value); err != nil {
+		return 0, err
+	}
+
+	var err error
+	key = db.encodeKVKey(key)
+
+	var n int64 = 1
+
+	t := db.kvBatch
+
+	t.Lock()
+	defer t.Unlock()
+
+	if v, err := db.bucket.Get(key); err != nil {
+		return 0, err
+	} else if v != nil {
+		n = 0
+	} else {
+		t.Put(key, value)
+
+		//todo binlog
+
+		err = t.Commit()
+	}
+
+	return n, err
+}
+
+func (db *DB) flush() (drop int64, err error) {
+	t := db.kvBatch
+	t.Lock()
+	defer t.Unlock()
+	return db.flushType(t, KVType)
+}
+
+//if inclusive is true, scan range [key, inf) else (key, inf)
+func (db *DB) Scan(key []byte, count int, inclusive bool, match string) ([][]byte, error) {
+	return db.scan(KVType, key, count, inclusive, match)
+}
+
+func (db *DB) Expire(key []byte, duration int64) (int64, error) {
+	if duration <= 0 {
+		return 0, errExpireValue
+	}
+
+	return db.setExpireAt(key, time.Now().Unix()+duration)
+}
+
+func (db *DB) ExpireAt(key []byte, when int64) (int64, error) {
+	if when <= time.Now().Unix() {
+		return 0, errExpireValue
+	}
+
+	return db.setExpireAt(key, when)
+}
+
+func (db *DB) TTL(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return -1, err
+	}
+
+	return db.ttl(KVType, key)
+}
+
+func (db *DB) Persist(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.kvBatch
+	t.Lock()
+	defer t.Unlock()
+	n, err := db.rmExpire(t, KVType, key)
+	if err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+	return n, err
+}
+
+func (db *DB) Lock() {
+	t := db.kvBatch
+	t.Lock()
+}
+
+func (db *DB) Remove(key []byte) bool {
+	if len(key) == 0 {
+		return false
+	}
+	t := db.kvBatch
+	t.Delete(db.encodeKVKey(key))
+	_, err := db.rmExpire(t, KVType, key)
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+func (db *DB) Commit() error {
+	t := db.kvBatch
+	return t.Commit()
+}
+
+func (db *DB) Unlock() {
+	t := db.kvBatch
+	t.Unlock()
+}

vendor/github.com/lunny/nodb/t_list.go

@@ -0,0 +1,492 @@
+package nodb
+
+import (
+	"encoding/binary"
+	"errors"
+	"time"
+
+	"github.com/lunny/nodb/store"
+)
+
+const (
+	listHeadSeq int32 = 1
+	listTailSeq int32 = 2
+
+	listMinSeq     int32 = 1000
+	listMaxSeq     int32 = 1<<31 - 1000
+	listInitialSeq int32 = listMinSeq + (listMaxSeq-listMinSeq)/2
+)
+
+var errLMetaKey = errors.New("invalid lmeta key")
+var errListKey = errors.New("invalid list key")
+var errListSeq = errors.New("invalid list sequence, overflow")
+
+func (db *DB) lEncodeMetaKey(key []byte) []byte {
+	buf := make([]byte, len(key)+2)
+	buf[0] = db.index
+	buf[1] = LMetaType
+
+	copy(buf[2:], key)
+	return buf
+}
+
+func (db *DB) lDecodeMetaKey(ek []byte) ([]byte, error) {
+	if len(ek) < 2 || ek[0] != db.index || ek[1] != LMetaType {
+		return nil, errLMetaKey
+	}
+
+	return ek[2:], nil
+}
+
+func (db *DB) lEncodeListKey(key []byte, seq int32) []byte {
+	buf := make([]byte, len(key)+8)
+
+	pos := 0
+	buf[pos] = db.index
+	pos++
+	buf[pos] = ListType
+	pos++
+
+	binary.BigEndian.PutUint16(buf[pos:], uint16(len(key)))
+	pos += 2
+
+	copy(buf[pos:], key)
+	pos += len(key)
+
+	binary.BigEndian.PutUint32(buf[pos:], uint32(seq))
+
+	return buf
+}
+
+func (db *DB) lDecodeListKey(ek []byte) (key []byte, seq int32, err error) {
+	if len(ek) < 8 || ek[0] != db.index || ek[1] != ListType {
+		err = errListKey
+		return
+	}
+
+	keyLen := int(binary.BigEndian.Uint16(ek[2:]))
+	if keyLen+8 != len(ek) {
+		err = errListKey
+		return
+	}
+
+	key = ek[4 : 4+keyLen]
+	seq = int32(binary.BigEndian.Uint32(ek[4+keyLen:]))
+	return
+}
+
+func (db *DB) lpush(key []byte, whereSeq int32, args ...[]byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	var headSeq int32
+	var tailSeq int32
+	var size int32
+	var err error
+
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+
+	metaKey := db.lEncodeMetaKey(key)
+	headSeq, tailSeq, size, err = db.lGetMeta(nil, metaKey)
+	if err != nil {
+		return 0, err
+	}
+
+	var pushCnt int = len(args)
+	if pushCnt == 0 {
+		return int64(size), nil
+	}
+
+	var seq int32 = headSeq
+	var delta int32 = -1
+	if whereSeq == listTailSeq {
+		seq = tailSeq
+		delta = 1
+	}
+
+	//	append elements
+	if size > 0 {
+		seq += delta
+	}
+
+	for i := 0; i < pushCnt; i++ {
+		ek := db.lEncodeListKey(key, seq+int32(i)*delta)
+		t.Put(ek, args[i])
+	}
+
+	seq += int32(pushCnt-1) * delta
+	if seq <= listMinSeq || seq >= listMaxSeq {
+		return 0, errListSeq
+	}
+
+	//	set meta info
+	if whereSeq == listHeadSeq {
+		headSeq = seq
+	} else {
+		tailSeq = seq
+	}
+
+	db.lSetMeta(metaKey, headSeq, tailSeq)
+
+	err = t.Commit()
+	return int64(size) + int64(pushCnt), err
+}
+
+func (db *DB) lpop(key []byte, whereSeq int32) ([]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+
+	var headSeq int32
+	var tailSeq int32
+	var err error
+
+	metaKey := db.lEncodeMetaKey(key)
+	headSeq, tailSeq, _, err = db.lGetMeta(nil, metaKey)
+	if err != nil {
+		return nil, err
+	}
+
+	var value []byte
+
+	var seq int32 = headSeq
+	if whereSeq == listTailSeq {
+		seq = tailSeq
+	}
+
+	itemKey := db.lEncodeListKey(key, seq)
+	value, err = db.bucket.Get(itemKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if whereSeq == listHeadSeq {
+		headSeq += 1
+	} else {
+		tailSeq -= 1
+	}
+
+	t.Delete(itemKey)
+	size := db.lSetMeta(metaKey, headSeq, tailSeq)
+	if size == 0 {
+		db.rmExpire(t, HashType, key)
+	}
+
+	err = t.Commit()
+	return value, err
+}
+
+//	ps : here just focus on deleting the list data,
+//		 any other likes expire is ignore.
+func (db *DB) lDelete(t *batch, key []byte) int64 {
+	mk := db.lEncodeMetaKey(key)
+
+	var headSeq int32
+	var tailSeq int32
+	var err error
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	headSeq, tailSeq, _, err = db.lGetMeta(it, mk)
+	if err != nil {
+		return 0
+	}
+
+	var num int64 = 0
+	startKey := db.lEncodeListKey(key, headSeq)
+	stopKey := db.lEncodeListKey(key, tailSeq)
+
+	rit := store.NewRangeIterator(it, &store.Range{startKey, stopKey, store.RangeClose})
+	for ; rit.Valid(); rit.Next() {
+		t.Delete(rit.RawKey())
+		num++
+	}
+
+	t.Delete(mk)
+
+	return num
+}
+
+func (db *DB) lGetMeta(it *store.Iterator, ek []byte) (headSeq int32, tailSeq int32, size int32, err error) {
+	var v []byte
+	if it != nil {
+		v = it.Find(ek)
+	} else {
+		v, err = db.bucket.Get(ek)
+	}
+	if err != nil {
+		return
+	} else if v == nil {
+		headSeq = listInitialSeq
+		tailSeq = listInitialSeq
+		size = 0
+		return
+	} else {
+		headSeq = int32(binary.LittleEndian.Uint32(v[0:4]))
+		tailSeq = int32(binary.LittleEndian.Uint32(v[4:8]))
+		size = tailSeq - headSeq + 1
+	}
+	return
+}
+
+func (db *DB) lSetMeta(ek []byte, headSeq int32, tailSeq int32) int32 {
+	t := db.listBatch
+
+	var size int32 = tailSeq - headSeq + 1
+	if size < 0 {
+		//	todo : log error + panic
+	} else if size == 0 {
+		t.Delete(ek)
+	} else {
+		buf := make([]byte, 8)
+
+		binary.LittleEndian.PutUint32(buf[0:4], uint32(headSeq))
+		binary.LittleEndian.PutUint32(buf[4:8], uint32(tailSeq))
+
+		t.Put(ek, buf)
+	}
+
+	return size
+}
+
+func (db *DB) lExpireAt(key []byte, when int64) (int64, error) {
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+
+	if llen, err := db.LLen(key); err != nil || llen == 0 {
+		return 0, err
+	} else {
+		db.expireAt(t, ListType, key, when)
+		if err := t.Commit(); err != nil {
+			return 0, err
+		}
+	}
+	return 1, nil
+}
+
+func (db *DB) LIndex(key []byte, index int32) ([]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	var seq int32
+	var headSeq int32
+	var tailSeq int32
+	var err error
+
+	metaKey := db.lEncodeMetaKey(key)
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	headSeq, tailSeq, _, err = db.lGetMeta(it, metaKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if index >= 0 {
+		seq = headSeq + index
+	} else {
+		seq = tailSeq + index + 1
+	}
+
+	sk := db.lEncodeListKey(key, seq)
+	v := it.Find(sk)
+
+	return v, nil
+}
+
+func (db *DB) LLen(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	ek := db.lEncodeMetaKey(key)
+	_, _, size, err := db.lGetMeta(nil, ek)
+	return int64(size), err
+}
+
+func (db *DB) LPop(key []byte) ([]byte, error) {
+	return db.lpop(key, listHeadSeq)
+}
+
+func (db *DB) LPush(key []byte, arg1 []byte, args ...[]byte) (int64, error) {
+	var argss = [][]byte{arg1}
+	argss = append(argss, args...)
+	return db.lpush(key, listHeadSeq, argss...)
+}
+
+func (db *DB) LRange(key []byte, start int32, stop int32) ([][]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	var headSeq int32
+	var llen int32
+	var err error
+
+	metaKey := db.lEncodeMetaKey(key)
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	if headSeq, _, llen, err = db.lGetMeta(it, metaKey); err != nil {
+		return nil, err
+	}
+
+	if start < 0 {
+		start = llen + start
+	}
+	if stop < 0 {
+		stop = llen + stop
+	}
+	if start < 0 {
+		start = 0
+	}
+
+	if start > stop || start >= llen {
+		return [][]byte{}, nil
+	}
+
+	if stop >= llen {
+		stop = llen - 1
+	}
+
+	limit := (stop - start) + 1
+	headSeq += start
+
+	v := make([][]byte, 0, limit)
+
+	startKey := db.lEncodeListKey(key, headSeq)
+	rit := store.NewRangeLimitIterator(it,
+		&store.Range{
+			Min:  startKey,
+			Max:  nil,
+			Type: store.RangeClose},
+		&store.Limit{
+			Offset: 0,
+			Count:  int(limit)})
+
+	for ; rit.Valid(); rit.Next() {
+		v = append(v, rit.Value())
+	}
+
+	return v, nil
+}
+
+func (db *DB) RPop(key []byte) ([]byte, error) {
+	return db.lpop(key, listTailSeq)
+}
+
+func (db *DB) RPush(key []byte, arg1 []byte, args ...[]byte) (int64, error) {
+	var argss = [][]byte{arg1}
+	argss = append(argss, args...)
+	return db.lpush(key, listTailSeq, argss...)
+}
+
+func (db *DB) LClear(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+
+	num := db.lDelete(t, key)
+	db.rmExpire(t, ListType, key)
+
+	err := t.Commit()
+	return num, err
+}
+
+func (db *DB) LMclear(keys ...[]byte) (int64, error) {
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+
+	for _, key := range keys {
+		if err := checkKeySize(key); err != nil {
+			return 0, err
+		}
+
+		db.lDelete(t, key)
+		db.rmExpire(t, ListType, key)
+
+	}
+
+	err := t.Commit()
+	return int64(len(keys)), err
+}
+
+func (db *DB) lFlush() (drop int64, err error) {
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+	return db.flushType(t, ListType)
+}
+
+func (db *DB) LExpire(key []byte, duration int64) (int64, error) {
+	if duration <= 0 {
+		return 0, errExpireValue
+	}
+
+	return db.lExpireAt(key, time.Now().Unix()+duration)
+}
+
+func (db *DB) LExpireAt(key []byte, when int64) (int64, error) {
+	if when <= time.Now().Unix() {
+		return 0, errExpireValue
+	}
+
+	return db.lExpireAt(key, when)
+}
+
+func (db *DB) LTTL(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return -1, err
+	}
+
+	return db.ttl(ListType, key)
+}
+
+func (db *DB) LPersist(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.listBatch
+	t.Lock()
+	defer t.Unlock()
+
+	n, err := db.rmExpire(t, ListType, key)
+	if err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+	return n, err
+}
+
+func (db *DB) LScan(key []byte, count int, inclusive bool, match string) ([][]byte, error) {
+	return db.scan(LMetaType, key, count, inclusive, match)
+}
+
+func (db *DB) lEncodeMinKey() []byte {
+	return db.lEncodeMetaKey(nil)
+}
+
+func (db *DB) lEncodeMaxKey() []byte {
+	ek := db.lEncodeMetaKey(nil)
+	ek[len(ek)-1] = LMetaType + 1
+	return ek
+}

vendor/github.com/lunny/nodb/t_set.go

@@ -0,0 +1,601 @@
+package nodb
+
+import (
+	"encoding/binary"
+	"errors"
+	"time"
+
+	"github.com/lunny/nodb/store"
+)
+
+var errSetKey = errors.New("invalid set key")
+var errSSizeKey = errors.New("invalid ssize key")
+
+const (
+	setStartSep byte = ':'
+	setStopSep  byte = setStartSep + 1
+	UnionType   byte = 51
+	DiffType    byte = 52
+	InterType   byte = 53
+)
+
+func checkSetKMSize(key []byte, member []byte) error {
+	if len(key) > MaxKeySize || len(key) == 0 {
+		return errKeySize
+	} else if len(member) > MaxSetMemberSize || len(member) == 0 {
+		return errSetMemberSize
+	}
+	return nil
+}
+
+func (db *DB) sEncodeSizeKey(key []byte) []byte {
+	buf := make([]byte, len(key)+2)
+
+	buf[0] = db.index
+	buf[1] = SSizeType
+
+	copy(buf[2:], key)
+	return buf
+}
+
+func (db *DB) sDecodeSizeKey(ek []byte) ([]byte, error) {
+	if len(ek) < 2 || ek[0] != db.index || ek[1] != SSizeType {
+		return nil, errSSizeKey
+	}
+
+	return ek[2:], nil
+}
+
+func (db *DB) sEncodeSetKey(key []byte, member []byte) []byte {
+	buf := make([]byte, len(key)+len(member)+1+1+2+1)
+
+	pos := 0
+	buf[pos] = db.index
+	pos++
+	buf[pos] = SetType
+	pos++
+
+	binary.BigEndian.PutUint16(buf[pos:], uint16(len(key)))
+	pos += 2
+
+	copy(buf[pos:], key)
+	pos += len(key)
+
+	buf[pos] = setStartSep
+	pos++
+	copy(buf[pos:], member)
+
+	return buf
+}
+
+func (db *DB) sDecodeSetKey(ek []byte) ([]byte, []byte, error) {
+	if len(ek) < 5 || ek[0] != db.index || ek[1] != SetType {
+		return nil, nil, errSetKey
+	}
+
+	pos := 2
+	keyLen := int(binary.BigEndian.Uint16(ek[pos:]))
+	pos += 2
+
+	if keyLen+5 > len(ek) {
+		return nil, nil, errSetKey
+	}
+
+	key := ek[pos : pos+keyLen]
+	pos += keyLen
+
+	if ek[pos] != hashStartSep {
+		return nil, nil, errSetKey
+	}
+
+	pos++
+	member := ek[pos:]
+	return key, member, nil
+}
+
+func (db *DB) sEncodeStartKey(key []byte) []byte {
+	return db.sEncodeSetKey(key, nil)
+}
+
+func (db *DB) sEncodeStopKey(key []byte) []byte {
+	k := db.sEncodeSetKey(key, nil)
+
+	k[len(k)-1] = setStopSep
+
+	return k
+}
+
+func (db *DB) sFlush() (drop int64, err error) {
+
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	return db.flushType(t, SetType)
+}
+
+func (db *DB) sDelete(t *batch, key []byte) int64 {
+	sk := db.sEncodeSizeKey(key)
+	start := db.sEncodeStartKey(key)
+	stop := db.sEncodeStopKey(key)
+
+	var num int64 = 0
+	it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1)
+	for ; it.Valid(); it.Next() {
+		t.Delete(it.RawKey())
+		num++
+	}
+
+	it.Close()
+
+	t.Delete(sk)
+	return num
+}
+
+func (db *DB) sIncrSize(key []byte, delta int64) (int64, error) {
+	t := db.setBatch
+	sk := db.sEncodeSizeKey(key)
+
+	var err error
+	var size int64 = 0
+	if size, err = Int64(db.bucket.Get(sk)); err != nil {
+		return 0, err
+	} else {
+		size += delta
+		if size <= 0 {
+			size = 0
+			t.Delete(sk)
+			db.rmExpire(t, SetType, key)
+		} else {
+			t.Put(sk, PutInt64(size))
+		}
+	}
+
+	return size, nil
+}
+
+func (db *DB) sExpireAt(key []byte, when int64) (int64, error) {
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	if scnt, err := db.SCard(key); err != nil || scnt == 0 {
+		return 0, err
+	} else {
+		db.expireAt(t, SetType, key, when)
+		if err := t.Commit(); err != nil {
+			return 0, err
+		}
+
+	}
+
+	return 1, nil
+}
+
+func (db *DB) sSetItem(key []byte, member []byte) (int64, error) {
+	t := db.setBatch
+	ek := db.sEncodeSetKey(key, member)
+
+	var n int64 = 1
+	if v, _ := db.bucket.Get(ek); v != nil {
+		n = 0
+	} else {
+		if _, err := db.sIncrSize(key, 1); err != nil {
+			return 0, err
+		}
+	}
+
+	t.Put(ek, nil)
+	return n, nil
+}
+
+func (db *DB) SAdd(key []byte, args ...[]byte) (int64, error) {
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	var err error
+	var ek []byte
+	var num int64 = 0
+	for i := 0; i < len(args); i++ {
+		if err := checkSetKMSize(key, args[i]); err != nil {
+			return 0, err
+		}
+
+		ek = db.sEncodeSetKey(key, args[i])
+
+		if v, err := db.bucket.Get(ek); err != nil {
+			return 0, err
+		} else if v == nil {
+			num++
+		}
+
+		t.Put(ek, nil)
+	}
+
+	if _, err = db.sIncrSize(key, num); err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+	return num, err
+
+}
+
+func (db *DB) SCard(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	sk := db.sEncodeSizeKey(key)
+
+	return Int64(db.bucket.Get(sk))
+}
+
+func (db *DB) sDiffGeneric(keys ...[]byte) ([][]byte, error) {
+	destMap := make(map[string]bool)
+
+	members, err := db.SMembers(keys[0])
+	if err != nil {
+		return nil, err
+	}
+
+	for _, m := range members {
+		destMap[String(m)] = true
+	}
+
+	for _, k := range keys[1:] {
+		members, err := db.SMembers(k)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, m := range members {
+			if _, ok := destMap[String(m)]; !ok {
+				continue
+			} else if ok {
+				delete(destMap, String(m))
+			}
+		}
+		// O - A = O, O is zero set.
+		if len(destMap) == 0 {
+			return nil, nil
+		}
+	}
+
+	slice := make([][]byte, len(destMap))
+	idx := 0
+	for k, v := range destMap {
+		if !v {
+			continue
+		}
+		slice[idx] = []byte(k)
+		idx++
+	}
+
+	return slice, nil
+}
+
+func (db *DB) SDiff(keys ...[]byte) ([][]byte, error) {
+	v, err := db.sDiffGeneric(keys...)
+	return v, err
+}
+
+func (db *DB) SDiffStore(dstKey []byte, keys ...[]byte) (int64, error) {
+	n, err := db.sStoreGeneric(dstKey, DiffType, keys...)
+	return n, err
+}
+
+func (db *DB) sInterGeneric(keys ...[]byte) ([][]byte, error) {
+	destMap := make(map[string]bool)
+
+	members, err := db.SMembers(keys[0])
+	if err != nil {
+		return nil, err
+	}
+
+	for _, m := range members {
+		destMap[String(m)] = true
+	}
+
+	for _, key := range keys[1:] {
+		if err := checkKeySize(key); err != nil {
+			return nil, err
+		}
+
+		members, err := db.SMembers(key)
+		if err != nil {
+			return nil, err
+		} else if len(members) == 0 {
+			return nil, err
+		}
+
+		tempMap := make(map[string]bool)
+		for _, member := range members {
+			if err := checkKeySize(member); err != nil {
+				return nil, err
+			}
+			if _, ok := destMap[String(member)]; ok {
+				tempMap[String(member)] = true //mark this item as selected
+			}
+		}
+		destMap = tempMap //reduce the size of the result set
+		if len(destMap) == 0 {
+			return nil, nil
+		}
+	}
+
+	slice := make([][]byte, len(destMap))
+	idx := 0
+	for k, v := range destMap {
+		if !v {
+			continue
+		}
+
+		slice[idx] = []byte(k)
+		idx++
+	}
+
+	return slice, nil
+
+}
+
+func (db *DB) SInter(keys ...[]byte) ([][]byte, error) {
+	v, err := db.sInterGeneric(keys...)
+	return v, err
+
+}
+
+func (db *DB) SInterStore(dstKey []byte, keys ...[]byte) (int64, error) {
+	n, err := db.sStoreGeneric(dstKey, InterType, keys...)
+	return n, err
+}
+
+func (db *DB) SIsMember(key []byte, member []byte) (int64, error) {
+	ek := db.sEncodeSetKey(key, member)
+
+	var n int64 = 1
+	if v, err := db.bucket.Get(ek); err != nil {
+		return 0, err
+	} else if v == nil {
+		n = 0
+	}
+	return n, nil
+}
+
+func (db *DB) SMembers(key []byte) ([][]byte, error) {
+	if err := checkKeySize(key); err != nil {
+		return nil, err
+	}
+
+	start := db.sEncodeStartKey(key)
+	stop := db.sEncodeStopKey(key)
+
+	v := make([][]byte, 0, 16)
+
+	it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1)
+	for ; it.Valid(); it.Next() {
+		_, m, err := db.sDecodeSetKey(it.Key())
+		if err != nil {
+			return nil, err
+		}
+
+		v = append(v, m)
+	}
+
+	it.Close()
+
+	return v, nil
+}
+
+func (db *DB) SRem(key []byte, args ...[]byte) (int64, error) {
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	var ek []byte
+	var v []byte
+	var err error
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	var num int64 = 0
+	for i := 0; i < len(args); i++ {
+		if err := checkSetKMSize(key, args[i]); err != nil {
+			return 0, err
+		}
+
+		ek = db.sEncodeSetKey(key, args[i])
+
+		v = it.RawFind(ek)
+		if v == nil {
+			continue
+		} else {
+			num++
+			t.Delete(ek)
+		}
+	}
+
+	if _, err = db.sIncrSize(key, -num); err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+	return num, err
+
+}
+
+func (db *DB) sUnionGeneric(keys ...[]byte) ([][]byte, error) {
+	dstMap := make(map[string]bool)
+
+	for _, key := range keys {
+		if err := checkKeySize(key); err != nil {
+			return nil, err
+		}
+
+		members, err := db.SMembers(key)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, member := range members {
+			dstMap[String(member)] = true
+		}
+	}
+
+	slice := make([][]byte, len(dstMap))
+	idx := 0
+	for k, v := range dstMap {
+		if !v {
+			continue
+		}
+		slice[idx] = []byte(k)
+		idx++
+	}
+
+	return slice, nil
+}
+
+func (db *DB) SUnion(keys ...[]byte) ([][]byte, error) {
+	v, err := db.sUnionGeneric(keys...)
+	return v, err
+}
+
+func (db *DB) SUnionStore(dstKey []byte, keys ...[]byte) (int64, error) {
+	n, err := db.sStoreGeneric(dstKey, UnionType, keys...)
+	return n, err
+}
+
+func (db *DB) sStoreGeneric(dstKey []byte, optType byte, keys ...[]byte) (int64, error) {
+	if err := checkKeySize(dstKey); err != nil {
+		return 0, err
+	}
+
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	db.sDelete(t, dstKey)
+
+	var err error
+	var ek []byte
+	var v [][]byte
+
+	switch optType {
+	case UnionType:
+		v, err = db.sUnionGeneric(keys...)
+	case DiffType:
+		v, err = db.sDiffGeneric(keys...)
+	case InterType:
+		v, err = db.sInterGeneric(keys...)
+	}
+
+	if err != nil {
+		return 0, err
+	}
+
+	for _, m := range v {
+		if err := checkSetKMSize(dstKey, m); err != nil {
+			return 0, err
+		}
+
+		ek = db.sEncodeSetKey(dstKey, m)
+
+		if _, err := db.bucket.Get(ek); err != nil {
+			return 0, err
+		}
+
+		t.Put(ek, nil)
+	}
+
+	var num = int64(len(v))
+	sk := db.sEncodeSizeKey(dstKey)
+	t.Put(sk, PutInt64(num))
+
+	if err = t.Commit(); err != nil {
+		return 0, err
+	}
+	return num, nil
+}
+
+func (db *DB) SClear(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	num := db.sDelete(t, key)
+	db.rmExpire(t, SetType, key)
+
+	err := t.Commit()
+	return num, err
+}
+
+func (db *DB) SMclear(keys ...[]byte) (int64, error) {
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	for _, key := range keys {
+		if err := checkKeySize(key); err != nil {
+			return 0, err
+		}
+
+		db.sDelete(t, key)
+		db.rmExpire(t, SetType, key)
+	}
+
+	err := t.Commit()
+	return int64(len(keys)), err
+}
+
+func (db *DB) SExpire(key []byte, duration int64) (int64, error) {
+	if duration <= 0 {
+		return 0, errExpireValue
+	}
+
+	return db.sExpireAt(key, time.Now().Unix()+duration)
+
+}
+
+func (db *DB) SExpireAt(key []byte, when int64) (int64, error) {
+	if when <= time.Now().Unix() {
+		return 0, errExpireValue
+	}
+
+	return db.sExpireAt(key, when)
+
+}
+
+func (db *DB) STTL(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return -1, err
+	}
+
+	return db.ttl(SetType, key)
+}
+
+func (db *DB) SPersist(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.setBatch
+	t.Lock()
+	defer t.Unlock()
+
+	n, err := db.rmExpire(t, SetType, key)
+	if err != nil {
+		return 0, err
+	}
+	err = t.Commit()
+	return n, err
+}
+
+func (db *DB) SScan(key []byte, count int, inclusive bool, match string) ([][]byte, error) {
+	return db.scan(SSizeType, key, count, inclusive, match)
+}

vendor/github.com/lunny/nodb/t_ttl.go

@@ -0,0 +1,195 @@
+package nodb
+
+import (
+	"encoding/binary"
+	"errors"
+	"time"
+
+	"github.com/lunny/nodb/store"
+)
+
+var (
+	errExpMetaKey = errors.New("invalid expire meta key")
+	errExpTimeKey = errors.New("invalid expire time key")
+)
+
+type retireCallback func(*batch, []byte) int64
+
+type elimination struct {
+	db         *DB
+	exp2Tx     []*batch
+	exp2Retire []retireCallback
+}
+
+var errExpType = errors.New("invalid expire type")
+
+func (db *DB) expEncodeTimeKey(dataType byte, key []byte, when int64) []byte {
+	buf := make([]byte, len(key)+11)
+
+	buf[0] = db.index
+	buf[1] = ExpTimeType
+	buf[2] = dataType
+	pos := 3
+
+	binary.BigEndian.PutUint64(buf[pos:], uint64(when))
+	pos += 8
+
+	copy(buf[pos:], key)
+
+	return buf
+}
+
+func (db *DB) expEncodeMetaKey(dataType byte, key []byte) []byte {
+	buf := make([]byte, len(key)+3)
+
+	buf[0] = db.index
+	buf[1] = ExpMetaType
+	buf[2] = dataType
+	pos := 3
+
+	copy(buf[pos:], key)
+
+	return buf
+}
+
+func (db *DB) expDecodeMetaKey(mk []byte) (byte, []byte, error) {
+	if len(mk) <= 3 || mk[0] != db.index || mk[1] != ExpMetaType {
+		return 0, nil, errExpMetaKey
+	}
+
+	return mk[2], mk[3:], nil
+}
+
+func (db *DB) expDecodeTimeKey(tk []byte) (byte, []byte, int64, error) {
+	if len(tk) < 11 || tk[0] != db.index || tk[1] != ExpTimeType {
+		return 0, nil, 0, errExpTimeKey
+	}
+
+	return tk[2], tk[11:], int64(binary.BigEndian.Uint64(tk[3:])), nil
+}
+
+func (db *DB) expire(t *batch, dataType byte, key []byte, duration int64) {
+	db.expireAt(t, dataType, key, time.Now().Unix()+duration)
+}
+
+func (db *DB) expireAt(t *batch, dataType byte, key []byte, when int64) {
+	mk := db.expEncodeMetaKey(dataType, key)
+	tk := db.expEncodeTimeKey(dataType, key, when)
+
+	t.Put(tk, mk)
+	t.Put(mk, PutInt64(when))
+}
+
+func (db *DB) ttl(dataType byte, key []byte) (t int64, err error) {
+	mk := db.expEncodeMetaKey(dataType, key)
+
+	if t, err = Int64(db.bucket.Get(mk)); err != nil || t == 0 {
+		t = -1
+	} else {
+		t -= time.Now().Unix()
+		if t <= 0 {
+			t = -1
+		}
+		// if t == -1 : to remove ????
+	}
+
+	return t, err
+}
+
+func (db *DB) rmExpire(t *batch, dataType byte, key []byte) (int64, error) {
+	mk := db.expEncodeMetaKey(dataType, key)
+	if v, err := db.bucket.Get(mk); err != nil {
+		return 0, err
+	} else if v == nil {
+		return 0, nil
+	} else if when, err2 := Int64(v, nil); err2 != nil {
+		return 0, err2
+	} else {
+		tk := db.expEncodeTimeKey(dataType, key, when)
+		t.Delete(mk)
+		t.Delete(tk)
+		return 1, nil
+	}
+}
+
+func (db *DB) expFlush(t *batch, dataType byte) (err error) {
+	minKey := make([]byte, 3)
+	minKey[0] = db.index
+	minKey[1] = ExpTimeType
+	minKey[2] = dataType
+
+	maxKey := make([]byte, 3)
+	maxKey[0] = db.index
+	maxKey[1] = ExpMetaType
+	maxKey[2] = dataType + 1
+
+	_, err = db.flushRegion(t, minKey, maxKey)
+	err = t.Commit()
+	return
+}
+
+//////////////////////////////////////////////////////////
+//
+//////////////////////////////////////////////////////////
+
+func newEliminator(db *DB) *elimination {
+	eli := new(elimination)
+	eli.db = db
+	eli.exp2Tx = make([]*batch, maxDataType)
+	eli.exp2Retire = make([]retireCallback, maxDataType)
+	return eli
+}
+
+func (eli *elimination) regRetireContext(dataType byte, t *batch, onRetire retireCallback) {
+
+	//	todo .. need to ensure exist - mapExpMetaType[expType]
+
+	eli.exp2Tx[dataType] = t
+	eli.exp2Retire[dataType] = onRetire
+}
+
+//	call by outside ... (from *db to another *db)
+func (eli *elimination) active() {
+	now := time.Now().Unix()
+	db := eli.db
+	dbGet := db.bucket.Get
+
+	minKey := db.expEncodeTimeKey(NoneType, nil, 0)
+	maxKey := db.expEncodeTimeKey(maxDataType, nil, now)
+
+	it := db.bucket.RangeLimitIterator(minKey, maxKey, store.RangeROpen, 0, -1)
+	for ; it.Valid(); it.Next() {
+		tk := it.RawKey()
+		mk := it.RawValue()
+
+		dt, k, _, err := db.expDecodeTimeKey(tk)
+		if err != nil {
+			continue
+		}
+
+		t := eli.exp2Tx[dt]
+		onRetire := eli.exp2Retire[dt]
+		if tk == nil || onRetire == nil {
+			continue
+		}
+
+		t.Lock()
+
+		if exp, err := Int64(dbGet(mk)); err == nil {
+			// check expire again
+			if exp <= now {
+				onRetire(t, k)
+				t.Delete(tk)
+				t.Delete(mk)
+
+				t.Commit()
+			}
+
+		}
+
+		t.Unlock()
+	}
+	it.Close()
+
+	return
+}

vendor/github.com/lunny/nodb/t_zset.go

@@ -0,0 +1,943 @@
+package nodb
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"time"
+
+	"github.com/lunny/nodb/store"
+)
+
+const (
+	MinScore     int64 = -1<<63 + 1
+	MaxScore     int64 = 1<<63 - 1
+	InvalidScore int64 = -1 << 63
+
+	AggregateSum byte = 0
+	AggregateMin byte = 1
+	AggregateMax byte = 2
+)
+
+type ScorePair struct {
+	Score  int64
+	Member []byte
+}
+
+var errZSizeKey = errors.New("invalid zsize key")
+var errZSetKey = errors.New("invalid zset key")
+var errZScoreKey = errors.New("invalid zscore key")
+var errScoreOverflow = errors.New("zset score overflow")
+var errInvalidAggregate = errors.New("invalid aggregate")
+var errInvalidWeightNum = errors.New("invalid weight number")
+var errInvalidSrcKeyNum = errors.New("invalid src key number")
+
+const (
+	zsetNScoreSep    byte = '<'
+	zsetPScoreSep    byte = zsetNScoreSep + 1
+	zsetStopScoreSep byte = zsetPScoreSep + 1
+
+	zsetStartMemSep byte = ':'
+	zsetStopMemSep  byte = zsetStartMemSep + 1
+)
+
+func checkZSetKMSize(key []byte, member []byte) error {
+	if len(key) > MaxKeySize || len(key) == 0 {
+		return errKeySize
+	} else if len(member) > MaxZSetMemberSize || len(member) == 0 {
+		return errZSetMemberSize
+	}
+	return nil
+}
+
+func (db *DB) zEncodeSizeKey(key []byte) []byte {
+	buf := make([]byte, len(key)+2)
+	buf[0] = db.index
+	buf[1] = ZSizeType
+
+	copy(buf[2:], key)
+	return buf
+}
+
+func (db *DB) zDecodeSizeKey(ek []byte) ([]byte, error) {
+	if len(ek) < 2 || ek[0] != db.index || ek[1] != ZSizeType {
+		return nil, errZSizeKey
+	}
+
+	return ek[2:], nil
+}
+
+func (db *DB) zEncodeSetKey(key []byte, member []byte) []byte {
+	buf := make([]byte, len(key)+len(member)+5)
+
+	pos := 0
+	buf[pos] = db.index
+	pos++
+
+	buf[pos] = ZSetType
+	pos++
+
+	binary.BigEndian.PutUint16(buf[pos:], uint16(len(key)))
+	pos += 2
+
+	copy(buf[pos:], key)
+	pos += len(key)
+
+	buf[pos] = zsetStartMemSep
+	pos++
+
+	copy(buf[pos:], member)
+
+	return buf
+}
+
+func (db *DB) zDecodeSetKey(ek []byte) ([]byte, []byte, error) {
+	if len(ek) < 5 || ek[0] != db.index || ek[1] != ZSetType {
+		return nil, nil, errZSetKey
+	}
+
+	keyLen := int(binary.BigEndian.Uint16(ek[2:]))
+	if keyLen+5 > len(ek) {
+		return nil, nil, errZSetKey
+	}
+
+	key := ek[4 : 4+keyLen]
+
+	if ek[4+keyLen] != zsetStartMemSep {
+		return nil, nil, errZSetKey
+	}
+
+	member := ek[5+keyLen:]
+	return key, member, nil
+}
+
+func (db *DB) zEncodeStartSetKey(key []byte) []byte {
+	k := db.zEncodeSetKey(key, nil)
+	return k
+}
+
+func (db *DB) zEncodeStopSetKey(key []byte) []byte {
+	k := db.zEncodeSetKey(key, nil)
+	k[len(k)-1] = zsetStartMemSep + 1
+	return k
+}
+
+func (db *DB) zEncodeScoreKey(key []byte, member []byte, score int64) []byte {
+	buf := make([]byte, len(key)+len(member)+14)
+
+	pos := 0
+	buf[pos] = db.index
+	pos++
+
+	buf[pos] = ZScoreType
+	pos++
+
+	binary.BigEndian.PutUint16(buf[pos:], uint16(len(key)))
+	pos += 2
+
+	copy(buf[pos:], key)
+	pos += len(key)
+
+	if score < 0 {
+		buf[pos] = zsetNScoreSep
+	} else {
+		buf[pos] = zsetPScoreSep
+	}
+
+	pos++
+	binary.BigEndian.PutUint64(buf[pos:], uint64(score))
+	pos += 8
+
+	buf[pos] = zsetStartMemSep
+	pos++
+
+	copy(buf[pos:], member)
+	return buf
+}
+
+func (db *DB) zEncodeStartScoreKey(key []byte, score int64) []byte {
+	return db.zEncodeScoreKey(key, nil, score)
+}
+
+func (db *DB) zEncodeStopScoreKey(key []byte, score int64) []byte {
+	k := db.zEncodeScoreKey(key, nil, score)
+	k[len(k)-1] = zsetStopMemSep
+	return k
+}
+
+func (db *DB) zDecodeScoreKey(ek []byte) (key []byte, member []byte, score int64, err error) {
+	if len(ek) < 14 || ek[0] != db.index || ek[1] != ZScoreType {
+		err = errZScoreKey
+		return
+	}
+
+	keyLen := int(binary.BigEndian.Uint16(ek[2:]))
+	if keyLen+14 > len(ek) {
+		err = errZScoreKey
+		return
+	}
+
+	key = ek[4 : 4+keyLen]
+	pos := 4 + keyLen
+
+	if (ek[pos] != zsetNScoreSep) && (ek[pos] != zsetPScoreSep) {
+		err = errZScoreKey
+		return
+	}
+	pos++
+
+	score = int64(binary.BigEndian.Uint64(ek[pos:]))
+	pos += 8
+
+	if ek[pos] != zsetStartMemSep {
+		err = errZScoreKey
+		return
+	}
+
+	pos++
+
+	member = ek[pos:]
+	return
+}
+
+func (db *DB) zSetItem(t *batch, key []byte, score int64, member []byte) (int64, error) {
+	if score <= MinScore || score >= MaxScore {
+		return 0, errScoreOverflow
+	}
+
+	var exists int64 = 0
+	ek := db.zEncodeSetKey(key, member)
+
+	if v, err := db.bucket.Get(ek); err != nil {
+		return 0, err
+	} else if v != nil {
+		exists = 1
+
+		if s, err := Int64(v, err); err != nil {
+			return 0, err
+		} else {
+			sk := db.zEncodeScoreKey(key, member, s)
+			t.Delete(sk)
+		}
+	}
+
+	t.Put(ek, PutInt64(score))
+
+	sk := db.zEncodeScoreKey(key, member, score)
+	t.Put(sk, []byte{})
+
+	return exists, nil
+}
+
+func (db *DB) zDelItem(t *batch, key []byte, member []byte, skipDelScore bool) (int64, error) {
+	ek := db.zEncodeSetKey(key, member)
+	if v, err := db.bucket.Get(ek); err != nil {
+		return 0, err
+	} else if v == nil {
+		//not exists
+		return 0, nil
+	} else {
+		//exists
+		if !skipDelScore {
+			//we must del score
+			if s, err := Int64(v, err); err != nil {
+				return 0, err
+			} else {
+				sk := db.zEncodeScoreKey(key, member, s)
+				t.Delete(sk)
+			}
+		}
+	}
+
+	t.Delete(ek)
+
+	return 1, nil
+}
+
+func (db *DB) zDelete(t *batch, key []byte) int64 {
+	delMembCnt, _ := db.zRemRange(t, key, MinScore, MaxScore, 0, -1)
+	//	todo : log err
+	return delMembCnt
+}
+
+func (db *DB) zExpireAt(key []byte, when int64) (int64, error) {
+	t := db.zsetBatch
+	t.Lock()
+	defer t.Unlock()
+
+	if zcnt, err := db.ZCard(key); err != nil || zcnt == 0 {
+		return 0, err
+	} else {
+		db.expireAt(t, ZSetType, key, when)
+		if err := t.Commit(); err != nil {
+			return 0, err
+		}
+	}
+	return 1, nil
+}
+
+func (db *DB) ZAdd(key []byte, args ...ScorePair) (int64, error) {
+	if len(args) == 0 {
+		return 0, nil
+	}
+
+	t := db.zsetBatch
+	t.Lock()
+	defer t.Unlock()
+
+	var num int64 = 0
+	for i := 0; i < len(args); i++ {
+		score := args[i].Score
+		member := args[i].Member
+
+		if err := checkZSetKMSize(key, member); err != nil {
+			return 0, err
+		}
+
+		if n, err := db.zSetItem(t, key, score, member); err != nil {
+			return 0, err
+		} else if n == 0 {
+			//add new
+			num++
+		}
+	}
+
+	if _, err := db.zIncrSize(t, key, num); err != nil {
+		return 0, err
+	}
+
+	//todo add binlog
+	err := t.Commit()
+	return num, err
+}
+
+func (db *DB) zIncrSize(t *batch, key []byte, delta int64) (int64, error) {
+	sk := db.zEncodeSizeKey(key)
+
+	size, err := Int64(db.bucket.Get(sk))
+	if err != nil {
+		return 0, err
+	} else {
+		size += delta
+		if size <= 0 {
+			size = 0
+			t.Delete(sk)
+			db.rmExpire(t, ZSetType, key)
+		} else {
+			t.Put(sk, PutInt64(size))
+		}
+	}
+
+	return size, nil
+}
+
+func (db *DB) ZCard(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	sk := db.zEncodeSizeKey(key)
+	return Int64(db.bucket.Get(sk))
+}
+
+func (db *DB) ZScore(key []byte, member []byte) (int64, error) {
+	if err := checkZSetKMSize(key, member); err != nil {
+		return InvalidScore, err
+	}
+
+	var score int64 = InvalidScore
+
+	k := db.zEncodeSetKey(key, member)
+	if v, err := db.bucket.Get(k); err != nil {
+		return InvalidScore, err
+	} else if v == nil {
+		return InvalidScore, ErrScoreMiss
+	} else {
+		if score, err = Int64(v, nil); err != nil {
+			return InvalidScore, err
+		}
+	}
+
+	return score, nil
+}
+
+func (db *DB) ZRem(key []byte, members ...[]byte) (int64, error) {
+	if len(members) == 0 {
+		return 0, nil
+	}
+
+	t := db.zsetBatch
+	t.Lock()
+	defer t.Unlock()
+
+	var num int64 = 0
+	for i := 0; i < len(members); i++ {
+		if err := checkZSetKMSize(key, members[i]); err != nil {
+			return 0, err
+		}
+
+		if n, err := db.zDelItem(t, key, members[i], false); err != nil {
+			return 0, err
+		} else if n == 1 {
+			num++
+		}
+	}
+
+	if _, err := db.zIncrSize(t, key, -num); err != nil {
+		return 0, err
+	}
+
+	err := t.Commit()
+	return num, err
+}
+
+func (db *DB) ZIncrBy(key []byte, delta int64, member []byte) (int64, error) {
+	if err := checkZSetKMSize(key, member); err != nil {
+		return InvalidScore, err
+	}
+
+	t := db.zsetBatch
+	t.Lock()
+	defer t.Unlock()
+
+	ek := db.zEncodeSetKey(key, member)
+
+	var oldScore int64 = 0
+	v, err := db.bucket.Get(ek)
+	if err != nil {
+		return InvalidScore, err
+	} else if v == nil {
+		db.zIncrSize(t, key, 1)
+	} else {
+		if oldScore, err = Int64(v, err); err != nil {
+			return InvalidScore, err
+		}
+	}
+
+	newScore := oldScore + delta
+	if newScore >= MaxScore || newScore <= MinScore {
+		return InvalidScore, errScoreOverflow
+	}
+
+	sk := db.zEncodeScoreKey(key, member, newScore)
+	t.Put(sk, []byte{})
+	t.Put(ek, PutInt64(newScore))
+
+	if v != nil {
+		// so as to update score, we must delete the old one
+		oldSk := db.zEncodeScoreKey(key, member, oldScore)
+		t.Delete(oldSk)
+	}
+
+	err = t.Commit()
+	return newScore, err
+}
+
+func (db *DB) ZCount(key []byte, min int64, max int64) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+	minKey := db.zEncodeStartScoreKey(key, min)
+	maxKey := db.zEncodeStopScoreKey(key, max)
+
+	rangeType := store.RangeROpen
+
+	it := db.bucket.RangeLimitIterator(minKey, maxKey, rangeType, 0, -1)
+	var n int64 = 0
+	for ; it.Valid(); it.Next() {
+		n++
+	}
+	it.Close()
+
+	return n, nil
+}
+
+func (db *DB) zrank(key []byte, member []byte, reverse bool) (int64, error) {
+	if err := checkZSetKMSize(key, member); err != nil {
+		return 0, err
+	}
+
+	k := db.zEncodeSetKey(key, member)
+
+	it := db.bucket.NewIterator()
+	defer it.Close()
+
+	if v := it.Find(k); v == nil {
+		return -1, nil
+	} else {
+		if s, err := Int64(v, nil); err != nil {
+			return 0, err
+		} else {
+			var rit *store.RangeLimitIterator
+
+			sk := db.zEncodeScoreKey(key, member, s)
+
+			if !reverse {
+				minKey := db.zEncodeStartScoreKey(key, MinScore)
+
+				rit = store.NewRangeIterator(it, &store.Range{minKey, sk, store.RangeClose})
+			} else {
+				maxKey := db.zEncodeStopScoreKey(key, MaxScore)
+				rit = store.NewRevRangeIterator(it, &store.Range{sk, maxKey, store.RangeClose})
+			}
+
+			var lastKey []byte = nil
+			var n int64 = 0
+
+			for ; rit.Valid(); rit.Next() {
+				n++
+
+				lastKey = rit.BufKey(lastKey)
+			}
+
+			if _, m, _, err := db.zDecodeScoreKey(lastKey); err == nil && bytes.Equal(m, member) {
+				n--
+				return n, nil
+			}
+		}
+	}
+
+	return -1, nil
+}
+
+func (db *DB) zIterator(key []byte, min int64, max int64, offset int, count int, reverse bool) *store.RangeLimitIterator {
+	minKey := db.zEncodeStartScoreKey(key, min)
+	maxKey := db.zEncodeStopScoreKey(key, max)
+
+	if !reverse {
+		return db.bucket.RangeLimitIterator(minKey, maxKey, store.RangeClose, offset, count)
+	} else {
+		return db.bucket.RevRangeLimitIterator(minKey, maxKey, store.RangeClose, offset, count)
+	}
+}
+
+func (db *DB) zRemRange(t *batch, key []byte, min int64, max int64, offset int, count int) (int64, error) {
+	if len(key) > MaxKeySize {
+		return 0, errKeySize
+	}
+
+	it := db.zIterator(key, min, max, offset, count, false)
+	var num int64 = 0
+	for ; it.Valid(); it.Next() {
+		sk := it.RawKey()
+		_, m, _, err := db.zDecodeScoreKey(sk)
+		if err != nil {
+			continue
+		}
+
+		if n, err := db.zDelItem(t, key, m, true); err != nil {
+			return 0, err
+		} else if n == 1 {
+			num++
+		}
+
+		t.Delete(sk)
+	}
+	it.Close()
+
+	if _, err := db.zIncrSize(t, key, -num); err != nil {
+		return 0, err
+	}
+
+	return num, nil
+}
+
+func (db *DB) zRange(key []byte, min int64, max int64, offset int, count int, reverse bool) ([]ScorePair, error) {
+	if len(key) > MaxKeySize {
+		return nil, errKeySize
+	}
+
+	if offset < 0 {
+		return []ScorePair{}, nil
+	}
+
+	nv := 64
+	if count > 0 {
+		nv = count
+	}
+
+	v := make([]ScorePair, 0, nv)
+
+	var it *store.RangeLimitIterator
+
+	//if reverse and offset is 0, count < 0, we may use forward iterator then reverse
+	//because store iterator prev is slower than next
+	if !reverse || (offset == 0 && count < 0) {
+		it = db.zIterator(key, min, max, offset, count, false)
+	} else {
+		it = db.zIterator(key, min, max, offset, count, true)
+	}
+
+	for ; it.Valid(); it.Next() {
+		_, m, s, err := db.zDecodeScoreKey(it.Key())
+		//may be we will check key equal?
+		if err != nil {
+			continue
+		}
+
+		v = append(v, ScorePair{Member: m, Score: s})
+	}
+	it.Close()
+
+	if reverse && (offset == 0 && count < 0) {
+		for i, j := 0, len(v)-1; i < j; i, j = i+1, j-1 {
+			v[i], v[j] = v[j], v[i]
+		}
+	}
+
+	return v, nil
+}
+
+func (db *DB) zParseLimit(key []byte, start int, stop int) (offset int, count int, err error) {
+	if start < 0 || stop < 0 {
+		//refer redis implementation
+		var size int64
+		size, err = db.ZCard(key)
+		if err != nil {
+			return
+		}
+
+		llen := int(size)
+
+		if start < 0 {
+			start = llen + start
+		}
+		if stop < 0 {
+			stop = llen + stop
+		}
+
+		if start < 0 {
+			start = 0
+		}
+
+		if start >= llen {
+			offset = -1
+			return
+		}
+	}
+
+	if start > stop {
+		offset = -1
+		return
+	}
+
+	offset = start
+	count = (stop - start) + 1
+	return
+}
+
+func (db *DB) ZClear(key []byte) (int64, error) {
+	t := db.zsetBatch
+	t.Lock()
+	defer t.Unlock()
+
+	rmCnt, err := db.zRemRange(t, key, MinScore, MaxScore, 0, -1)
+	if err == nil {
+		err = t.Commit()
+	}
+
+	return rmCnt, err
+}
+
+func (db *DB) ZMclear(keys ...[]byte) (int64, error) {
+	t := db.zsetBatch
+	t.Lock()
+	defer t.Unlock()
+
+	for _, key := range keys {
+		if _, err := db.zRemRange(t, key, MinScore, MaxScore, 0, -1); err != nil {
+			return 0, err
+		}
+	}
+
+	err := t.Commit()
+
+	return int64(len(keys)), err
+}
+
+func (db *DB) ZRange(key []byte, start int, stop int) ([]ScorePair, error) {
+	return db.ZRangeGeneric(key, start, stop, false)
+}
+
+//min and max must be inclusive
+//if no limit, set offset = 0 and count = -1
+func (db *DB) ZRangeByScore(key []byte, min int64, max int64,
+	offset int, count int) ([]ScorePair, error) {
+	return db.ZRangeByScoreGeneric(key, min, max, offset, count, false)
+}
+
+func (db *DB) ZRank(key []byte, member []byte) (int64, error) {
+	return db.zrank(key, member, false)
+}
+
+func (db *DB) ZRemRangeByRank(key []byte, start int, stop int) (int64, error) {
+	offset, count, err := db.zParseLimit(key, start, stop)
+	if err != nil {
+		return 0, err
+	}
+
+	var rmCnt int64
+
+	t := db.zsetBatch
+	t.Lock()
+	defer t.Unlock()
+
+	rmCnt, err = db.zRemRange(t, key, MinScore, MaxScore, offset, count)
+	if err == nil {
+		err = t.Commit()
+	}
+
+	return rmCnt, err
+}
+
+//min and max must be inclusive
+func (db *DB) ZRemRangeByScore(key []byte, min int64, max int64) (int64, error) {
+	t := db.zsetBatch
+	t.Lock()
+	defer t.Unlock()
+
+	rmCnt, err := db.zRemRange(t, key, min, max, 0, -1)
+	if err == nil {
+		err = t.Commit()
+	}
+
+	return rmCnt, err
+}
+
+func (db *DB) ZRevRange(key []byte, start int, stop int) ([]ScorePair, error) {
+	return db.ZRangeGeneric(key, start, stop, true)
+}
+
+func (db *DB) ZRevRank(key []byte, member []byte) (int64, error) {
+	return db.zrank(key, member, true)
+}
+
+//min and max must be inclusive
+//if no limit, set offset = 0 and count = -1
+func (db *DB) ZRevRangeByScore(key []byte, min int64, max int64, offset int, count int) ([]ScorePair, error) {
+	return db.ZRangeByScoreGeneric(key, min, max, offset, count, true)
+}
+
+func (db *DB) ZRangeGeneric(key []byte, start int, stop int, reverse bool) ([]ScorePair, error) {
+	offset, count, err := db.zParseLimit(key, start, stop)
+	if err != nil {
+		return nil, err
+	}
+
+	return db.zRange(key, MinScore, MaxScore, offset, count, reverse)
+}
+
+//min and max must be inclusive
+//if no limit, set offset = 0 and count = -1
+func (db *DB) ZRangeByScoreGeneric(key []byte, min int64, max int64,
+	offset int, count int, reverse bool) ([]ScorePair, error) {
+
+	return db.zRange(key, min, max, offset, count, reverse)
+}
+
+func (db *DB) zFlush() (drop int64, err error) {
+	t := db.zsetBatch
+	t.Lock()
+	defer t.Unlock()
+	return db.flushType(t, ZSetType)
+}
+
+func (db *DB) ZExpire(key []byte, duration int64) (int64, error) {
+	if duration <= 0 {
+		return 0, errExpireValue
+	}
+
+	return db.zExpireAt(key, time.Now().Unix()+duration)
+}
+
+func (db *DB) ZExpireAt(key []byte, when int64) (int64, error) {
+	if when <= time.Now().Unix() {
+		return 0, errExpireValue
+	}
+
+	return db.zExpireAt(key, when)
+}
+
+func (db *DB) ZTTL(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return -1, err
+	}
+
+	return db.ttl(ZSetType, key)
+}
+
+func (db *DB) ZPersist(key []byte) (int64, error) {
+	if err := checkKeySize(key); err != nil {
+		return 0, err
+	}
+
+	t := db.zsetBatch
+	t.Lock()
+	defer t.Unlock()
+
+	n, err := db.rmExpire(t, ZSetType, key)
+	if err != nil {
+		return 0, err
+	}
+
+	err = t.Commit()
+	return n, err
+}
+
+func getAggregateFunc(aggregate byte) func(int64, int64) int64 {
+	switch aggregate {
+	case AggregateSum:
+		return func(a int64, b int64) int64 {
+			return a + b
+		}
+	case AggregateMax:
+		return func(a int64, b int64) int64 {
+			if a > b {
+				return a
+			}
+			return b
+		}
+	case AggregateMin:
+		return func(a int64, b int64) int64 {
+			if a > b {
+				return b
+			}
+			return a
+		}
+	}
+	return nil
+}
+
+func (db *DB) ZUnionStore(destKey []byte, srcKeys [][]byte, weights []int64, aggregate byte) (int64, error) {
+
+	var destMap = map[string]int64{}
+	aggregateFunc := getAggregateFunc(aggregate)
+	if aggregateFunc == nil {
+		return 0, errInvalidAggregate
+	}
+	if len(srcKeys) < 1 {
+		return 0, errInvalidSrcKeyNum
+	}
+	if weights != nil {
+		if len(srcKeys) != len(weights) {
+			return 0, errInvalidWeightNum
+		}
+	} else {
+		weights = make([]int64, len(srcKeys))
+		for i := 0; i < len(weights); i++ {
+			weights[i] = 1
+		}
+	}
+
+	for i, key := range srcKeys {
+		scorePairs, err := db.ZRange(key, 0, -1)
+		if err != nil {
+			return 0, err
+		}
+		for _, pair := range scorePairs {
+			if score, ok := destMap[String(pair.Member)]; !ok {
+				destMap[String(pair.Member)] = pair.Score * weights[i]
+			} else {
+				destMap[String(pair.Member)] = aggregateFunc(score, pair.Score*weights[i])
+			}
+		}
+	}
+
+	t := db.zsetBatch
+	t.Lock()
+	defer t.Unlock()
+
+	db.zDelete(t, destKey)
+
+	for member, score := range destMap {
+		if err := checkZSetKMSize(destKey, []byte(member)); err != nil {
+			return 0, err
+		}
+
+		if _, err := db.zSetItem(t, destKey, score, []byte(member)); err != nil {
+			return 0, err
+		}
+	}
+
+	var num = int64(len(destMap))
+	sk := db.zEncodeSizeKey(destKey)
+	t.Put(sk, PutInt64(num))
+
+	//todo add binlog
+	if err := t.Commit(); err != nil {
+		return 0, err
+	}
+	return num, nil
+}
+
+func (db *DB) ZInterStore(destKey []byte, srcKeys [][]byte, weights []int64, aggregate byte) (int64, error) {
+
+	aggregateFunc := getAggregateFunc(aggregate)
+	if aggregateFunc == nil {
+		return 0, errInvalidAggregate
+	}
+	if len(srcKeys) < 1 {
+		return 0, errInvalidSrcKeyNum
+	}
+	if weights != nil {
+		if len(srcKeys) != len(weights) {
+			return 0, errInvalidWeightNum
+		}
+	} else {
+		weights = make([]int64, len(srcKeys))
+		for i := 0; i < len(weights); i++ {
+			weights[i] = 1
+		}
+	}
+
+	var destMap = map[string]int64{}
+	scorePairs, err := db.ZRange(srcKeys[0], 0, -1)
+	if err != nil {
+		return 0, err
+	}
+	for _, pair := range scorePairs {
+		destMap[String(pair.Member)] = pair.Score * weights[0]
+	}
+
+	for i, key := range srcKeys[1:] {
+		scorePairs, err := db.ZRange(key, 0, -1)
+		if err != nil {
+			return 0, err
+		}
+		tmpMap := map[string]int64{}
+		for _, pair := range scorePairs {
+			if score, ok := destMap[String(pair.Member)]; ok {
+				tmpMap[String(pair.Member)] = aggregateFunc(score, pair.Score*weights[i+1])
+			}
+		}
+		destMap = tmpMap
+	}
+
+	t := db.zsetBatch
+	t.Lock()
+	defer t.Unlock()
+
+	db.zDelete(t, destKey)
+
+	for member, score := range destMap {
+		if err := checkZSetKMSize(destKey, []byte(member)); err != nil {
+			return 0, err
+		}
+		if _, err := db.zSetItem(t, destKey, score, []byte(member)); err != nil {
+			return 0, err
+		}
+	}
+
+	var num int64 = int64(len(destMap))
+	sk := db.zEncodeSizeKey(destKey)
+	t.Put(sk, PutInt64(num))
+	//todo add binlog
+	if err := t.Commit(); err != nil {
+		return 0, err
+	}
+	return num, nil
+}
+
+func (db *DB) ZScan(key []byte, count int, inclusive bool, match string) ([][]byte, error) {
+	return db.scan(ZSizeType, key, count, inclusive, match)
+}

vendor/github.com/lunny/nodb/tx.go

@@ -0,0 +1,113 @@
+package nodb
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/lunny/nodb/store"
+)
+
+var (
+	ErrNestTx = errors.New("nest transaction not supported")
+	ErrTxDone = errors.New("Transaction has already been committed or rolled back")
+)
+
+type Tx struct {
+	*DB
+
+	tx *store.Tx
+
+	logs [][]byte
+}
+
+func (db *DB) IsTransaction() bool {
+	return db.status == DBInTransaction
+}
+
+// Begin a transaction, it will block all other write operations before calling Commit or Rollback.
+// You must be very careful to prevent long-time transaction.
+func (db *DB) Begin() (*Tx, error) {
+	if db.IsTransaction() {
+		return nil, ErrNestTx
+	}
+
+	tx := new(Tx)
+
+	tx.DB = new(DB)
+	tx.DB.l = db.l
+
+	tx.l.wLock.Lock()
+
+	tx.DB.sdb = db.sdb
+
+	var err error
+	tx.tx, err = db.sdb.Begin()
+	if err != nil {
+		tx.l.wLock.Unlock()
+		return nil, err
+	}
+
+	tx.DB.bucket = tx.tx
+
+	tx.DB.status = DBInTransaction
+
+	tx.DB.index = db.index
+
+	tx.DB.kvBatch = tx.newBatch()
+	tx.DB.listBatch = tx.newBatch()
+	tx.DB.hashBatch = tx.newBatch()
+	tx.DB.zsetBatch = tx.newBatch()
+	tx.DB.binBatch = tx.newBatch()
+	tx.DB.setBatch = tx.newBatch()
+
+	return tx, nil
+}
+
+func (tx *Tx) Commit() error {
+	if tx.tx == nil {
+		return ErrTxDone
+	}
+
+	tx.l.commitLock.Lock()
+	err := tx.tx.Commit()
+	tx.tx = nil
+
+	if len(tx.logs) > 0 {
+		tx.l.binlog.Log(tx.logs...)
+	}
+
+	tx.l.commitLock.Unlock()
+
+	tx.l.wLock.Unlock()
+
+	tx.DB.bucket = nil
+
+	return err
+}
+
+func (tx *Tx) Rollback() error {
+	if tx.tx == nil {
+		return ErrTxDone
+	}
+
+	err := tx.tx.Rollback()
+	tx.tx = nil
+
+	tx.l.wLock.Unlock()
+	tx.DB.bucket = nil
+
+	return err
+}
+
+func (tx *Tx) newBatch() *batch {
+	return tx.l.newBatch(tx.tx.NewWriteBatch(), &txBatchLocker{}, tx)
+}
+
+func (tx *Tx) Select(index int) error {
+	if index < 0 || index >= int(MaxDBNumber) {
+		return fmt.Errorf("invalid db index %d", index)
+	}
+
+	tx.DB.index = uint8(index)
+	return nil
+}

vendor/github.com/lunny/nodb/util.go

@@ -0,0 +1,113 @@
+package nodb
+
+import (
+	"encoding/binary"
+	"errors"
+	"reflect"
+	"strconv"
+	"unsafe"
+)
+
+var errIntNumber = errors.New("invalid integer")
+
+// no copy to change slice to string
+// use your own risk
+func String(b []byte) (s string) {
+	pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	pstring := (*reflect.StringHeader)(unsafe.Pointer(&s))
+	pstring.Data = pbytes.Data
+	pstring.Len = pbytes.Len
+	return
+}
+
+// no copy to change string to slice
+// use your own risk
+func Slice(s string) (b []byte) {
+	pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	pstring := (*reflect.StringHeader)(unsafe.Pointer(&s))
+	pbytes.Data = pstring.Data
+	pbytes.Len = pstring.Len
+	pbytes.Cap = pstring.Len
+	return
+}
+
+func Int64(v []byte, err error) (int64, error) {
+	if err != nil {
+		return 0, err
+	} else if v == nil || len(v) == 0 {
+		return 0, nil
+	} else if len(v) != 8 {
+		return 0, errIntNumber
+	}
+
+	return int64(binary.LittleEndian.Uint64(v)), nil
+}
+
+func PutInt64(v int64) []byte {
+	var b []byte
+	pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b))
+	pbytes.Data = uintptr(unsafe.Pointer(&v))
+	pbytes.Len = 8
+	pbytes.Cap = 8
+	return b
+}
+
+func StrInt64(v []byte, err error) (int64, error) {
+	if err != nil {
+		return 0, err
+	} else if v == nil {
+		return 0, nil
+	} else {
+		return strconv.ParseInt(String(v), 10, 64)
+	}
+}
+
+func StrInt32(v []byte, err error) (int32, error) {
+	if err != nil {
+		return 0, err
+	} else if v == nil {
+		return 0, nil
+	} else {
+		res, err := strconv.ParseInt(String(v), 10, 32)
+		return int32(res), err
+	}
+}
+
+func StrInt8(v []byte, err error) (int8, error) {
+	if err != nil {
+		return 0, err
+	} else if v == nil {
+		return 0, nil
+	} else {
+		res, err := strconv.ParseInt(String(v), 10, 8)
+		return int8(res), err
+	}
+}
+
+func StrPutInt64(v int64) []byte {
+	return strconv.AppendInt(nil, v, 10)
+}
+
+func MinUInt32(a uint32, b uint32) uint32 {
+	if a > b {
+		return b
+	} else {
+		return a
+	}
+}
+
+func MaxUInt32(a uint32, b uint32) uint32 {
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+func MaxInt32(a int32, b int32) int32 {
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}

vendor/github.com/pkg/errors/LICENSE

@@ -0,0 +1,23 @@
+Copyright (c) 2015, Dave Cheney <dave@cheney.net>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/pkg/errors/errors.go

@@ -0,0 +1,282 @@
+// Package errors provides simple error handling primitives.
+//
+// The traditional error handling idiom in Go is roughly akin to
+//
+//     if err != nil {
+//             return err
+//     }
+//
+// which when applied recursively up the call stack results in error reports
+// without context or debugging information. The errors package allows
+// programmers to add context to the failure path in their code in a way
+// that does not destroy the original value of the error.
+//
+// Adding context to an error
+//
+// The errors.Wrap function returns a new error that adds context to the
+// original error by recording a stack trace at the point Wrap is called,
+// together with the supplied message. For example
+//
+//     _, err := ioutil.ReadAll(r)
+//     if err != nil {
+//             return errors.Wrap(err, "read failed")
+//     }
+//
+// If additional control is required, the errors.WithStack and
+// errors.WithMessage functions destructure errors.Wrap into its component
+// operations: annotating an error with a stack trace and with a message,
+// respectively.
+//
+// Retrieving the cause of an error
+//
+// Using errors.Wrap constructs a stack of errors, adding context to the
+// preceding error. Depending on the nature of the error it may be necessary
+// to reverse the operation of errors.Wrap to retrieve the original error
+// for inspection. Any error value which implements this interface
+//
+//     type causer interface {
+//             Cause() error
+//     }
+//
+// can be inspected by errors.Cause. errors.Cause will recursively retrieve
+// the topmost error that does not implement causer, which is assumed to be
+// the original cause. For example:
+//
+//     switch err := errors.Cause(err).(type) {
+//     case *MyError:
+//             // handle specifically
+//     default:
+//             // unknown error
+//     }
+//
+// Although the causer interface is not exported by this package, it is
+// considered a part of its stable public interface.
+//
+// Formatted printing of errors
+//
+// All error values returned from this package implement fmt.Formatter and can
+// be formatted by the fmt package. The following verbs are supported:
+//
+//     %s    print the error. If the error has a Cause it will be
+//           printed recursively.
+//     %v    see %s
+//     %+v   extended format. Each Frame of the error's StackTrace will
+//           be printed in detail.
+//
+// Retrieving the stack trace of an error or wrapper
+//
+// New, Errorf, Wrap, and Wrapf record a stack trace at the point they are
+// invoked. This information can be retrieved with the following interface:
+//
+//     type stackTracer interface {
+//             StackTrace() errors.StackTrace
+//     }
+//
+// The returned errors.StackTrace type is defined as
+//
+//     type StackTrace []Frame
+//
+// The Frame type represents a call site in the stack trace. Frame supports
+// the fmt.Formatter interface that can be used for printing information about
+// the stack trace of this error. For example:
+//
+//     if err, ok := err.(stackTracer); ok {
+//             for _, f := range err.StackTrace() {
+//                     fmt.Printf("%+s:%d", f)
+//             }
+//     }
+//
+// Although the stackTracer interface is not exported by this package, it is
+// considered a part of its stable public interface.
+//
+// See the documentation for Frame.Format for more details.
+package errors
+
+import (
+	"fmt"
+	"io"
+)
+
+// New returns an error with the supplied message.
+// New also records the stack trace at the point it was called.
+func New(message string) error {
+	return &fundamental{
+		msg:   message,
+		stack: callers(),
+	}
+}
+
+// Errorf formats according to a format specifier and returns the string
+// as a value that satisfies error.
+// Errorf also records the stack trace at the point it was called.
+func Errorf(format string, args ...interface{}) error {
+	return &fundamental{
+		msg:   fmt.Sprintf(format, args...),
+		stack: callers(),
+	}
+}
+
+// fundamental is an error that has a message and a stack, but no caller.
+type fundamental struct {
+	msg string
+	*stack
+}
+
+func (f *fundamental) Error() string { return f.msg }
+
+func (f *fundamental) Format(s fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		if s.Flag('+') {
+			io.WriteString(s, f.msg)
+			f.stack.Format(s, verb)
+			return
+		}
+		fallthrough
+	case 's':
+		io.WriteString(s, f.msg)
+	case 'q':
+		fmt.Fprintf(s, "%q", f.msg)
+	}
+}
+
+// WithStack annotates err with a stack trace at the point WithStack was called.
+// If err is nil, WithStack returns nil.
+func WithStack(err error) error {
+	if err == nil {
+		return nil
+	}
+	return &withStack{
+		err,
+		callers(),
+	}
+}
+
+type withStack struct {
+	error
+	*stack
+}
+
+func (w *withStack) Cause() error { return w.error }
+
+func (w *withStack) Format(s fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		if s.Flag('+') {
+			fmt.Fprintf(s, "%+v", w.Cause())
+			w.stack.Format(s, verb)
+			return
+		}
+		fallthrough
+	case 's':
+		io.WriteString(s, w.Error())
+	case 'q':
+		fmt.Fprintf(s, "%q", w.Error())
+	}
+}
+
+// Wrap returns an error annotating err with a stack trace
+// at the point Wrap is called, and the supplied message.
+// If err is nil, Wrap returns nil.
+func Wrap(err error, message string) error {
+	if err == nil {
+		return nil
+	}
+	err = &withMessage{
+		cause: err,
+		msg:   message,
+	}
+	return &withStack{
+		err,
+		callers(),
+	}
+}
+
+// Wrapf returns an error annotating err with a stack trace
+// at the point Wrapf is called, and the format specifier.
+// If err is nil, Wrapf returns nil.
+func Wrapf(err error, format string, args ...interface{}) error {
+	if err == nil {
+		return nil
+	}
+	err = &withMessage{
+		cause: err,
+		msg:   fmt.Sprintf(format, args...),
+	}
+	return &withStack{
+		err,
+		callers(),
+	}
+}
+
+// WithMessage annotates err with a new message.
+// If err is nil, WithMessage returns nil.
+func WithMessage(err error, message string) error {
+	if err == nil {
+		return nil
+	}
+	return &withMessage{
+		cause: err,
+		msg:   message,
+	}
+}
+
+// WithMessagef annotates err with the format specifier.
+// If err is nil, WithMessagef returns nil.
+func WithMessagef(err error, format string, args ...interface{}) error {
+	if err == nil {
+		return nil
+	}
+	return &withMessage{
+		cause: err,
+		msg:   fmt.Sprintf(format, args...),
+	}
+}
+
+type withMessage struct {
+	cause error
+	msg   string
+}
+
+func (w *withMessage) Error() string { return w.msg + ": " + w.cause.Error() }
+func (w *withMessage) Cause() error  { return w.cause }
+
+func (w *withMessage) Format(s fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		if s.Flag('+') {
+			fmt.Fprintf(s, "%+v\n", w.Cause())
+			io.WriteString(s, w.msg)
+			return
+		}
+		fallthrough
+	case 's', 'q':
+		io.WriteString(s, w.Error())
+	}
+}
+
+// Cause returns the underlying cause of the error, if possible.
+// An error value has a cause if it implements the following
+// interface:
+//
+//     type causer interface {
+//            Cause() error
+//     }
+//
+// If the error does not implement Cause, the original error will
+// be returned. If the error is nil, nil will be returned without further
+// investigation.
+func Cause(err error) error {
+	type causer interface {
+		Cause() error
+	}
+
+	for err != nil {
+		cause, ok := err.(causer)
+		if !ok {
+			break
+		}
+		err = cause.Cause()
+	}
+	return err
+}

vendor/github.com/pkg/errors/stack.go

@@ -0,0 +1,147 @@
+package errors
+
+import (
+	"fmt"
+	"io"
+	"path"
+	"runtime"
+	"strings"
+)
+
+// Frame represents a program counter inside a stack frame.
+type Frame uintptr
+
+// pc returns the program counter for this frame;
+// multiple frames may have the same PC value.
+func (f Frame) pc() uintptr { return uintptr(f) - 1 }
+
+// file returns the full path to the file that contains the
+// function for this Frame's pc.
+func (f Frame) file() string {
+	fn := runtime.FuncForPC(f.pc())
+	if fn == nil {
+		return "unknown"
+	}
+	file, _ := fn.FileLine(f.pc())
+	return file
+}
+
+// line returns the line number of source code of the
+// function for this Frame's pc.
+func (f Frame) line() int {
+	fn := runtime.FuncForPC(f.pc())
+	if fn == nil {
+		return 0
+	}
+	_, line := fn.FileLine(f.pc())
+	return line
+}
+
+// Format formats the frame according to the fmt.Formatter interface.
+//
+//    %s    source file
+//    %d    source line
+//    %n    function name
+//    %v    equivalent to %s:%d
+//
+// Format accepts flags that alter the printing of some verbs, as follows:
+//
+//    %+s   function name and path of source file relative to the compile time
+//          GOPATH separated by \n\t (<funcname>\n\t<path>)
+//    %+v   equivalent to %+s:%d
+func (f Frame) Format(s fmt.State, verb rune) {
+	switch verb {
+	case 's':
+		switch {
+		case s.Flag('+'):
+			pc := f.pc()
+			fn := runtime.FuncForPC(pc)
+			if fn == nil {
+				io.WriteString(s, "unknown")
+			} else {
+				file, _ := fn.FileLine(pc)
+				fmt.Fprintf(s, "%s\n\t%s", fn.Name(), file)
+			}
+		default:
+			io.WriteString(s, path.Base(f.file()))
+		}
+	case 'd':
+		fmt.Fprintf(s, "%d", f.line())
+	case 'n':
+		name := runtime.FuncForPC(f.pc()).Name()
+		io.WriteString(s, funcname(name))
+	case 'v':
+		f.Format(s, 's')
+		io.WriteString(s, ":")
+		f.Format(s, 'd')
+	}
+}
+
+// StackTrace is stack of Frames from innermost (newest) to outermost (oldest).
+type StackTrace []Frame
+
+// Format formats the stack of Frames according to the fmt.Formatter interface.
+//
+//    %s	lists source files for each Frame in the stack
+//    %v	lists the source file and line number for each Frame in the stack
+//
+// Format accepts flags that alter the printing of some verbs, as follows:
+//
+//    %+v   Prints filename, function, and line number for each Frame in the stack.
+func (st StackTrace) Format(s fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		switch {
+		case s.Flag('+'):
+			for _, f := range st {
+				fmt.Fprintf(s, "\n%+v", f)
+			}
+		case s.Flag('#'):
+			fmt.Fprintf(s, "%#v", []Frame(st))
+		default:
+			fmt.Fprintf(s, "%v", []Frame(st))
+		}
+	case 's':
+		fmt.Fprintf(s, "%s", []Frame(st))
+	}
+}
+
+// stack represents a stack of program counters.
+type stack []uintptr
+
+func (s *stack) Format(st fmt.State, verb rune) {
+	switch verb {
+	case 'v':
+		switch {
+		case st.Flag('+'):
+			for _, pc := range *s {
+				f := Frame(pc)
+				fmt.Fprintf(st, "\n%+v", f)
+			}
+		}
+	}
+}
+
+func (s *stack) StackTrace() StackTrace {
+	f := make([]Frame, len(*s))
+	for i := 0; i < len(f); i++ {
+		f[i] = Frame((*s)[i])
+	}
+	return f
+}
+
+func callers() *stack {
+	const depth = 32
+	var pcs [depth]uintptr
+	n := runtime.Callers(3, pcs[:])
+	var st stack = pcs[0:n]
+	return &st
+}
+
+// funcname removes the path prefix component of a function's name reported by func.Name().
+func funcname(name string) string {
+	i := strings.LastIndex(name, "/")
+	name = name[i+1:]
+	i = strings.Index(name, ".")
+	return name[i+1:]
+}

vendor/github.com/siddontang/go-snappy/AUTHORS

@@ -0,0 +1,12 @@
+# This is the official list of Snappy-Go authors for copyright purposes.
+# This file is distinct from the CONTRIBUTORS files.
+# See the latter for an explanation.
+
+# Names should be added to this file as
+#	Name or Organization <email address>
+# The email address is not required for organizations.
+
+# Please keep the list sorted.
+
+Google Inc.
+Jan Mercl <0xjnml@gmail.com>