diff --git a/bucket.go b/bucket.go index 785ad9bd5..f889b8109 100644 --- a/bucket.go +++ b/bucket.go @@ -1000,6 +1000,6 @@ func cloneBytes(v []byte) []byte { type BucketStructure struct { Name string `json:"name"` // name of the bucket - KeyN int `json:"keyN"` // number of key/value pairs Children []BucketStructure `json:"buckets,omitempty"` // child buckets + KeyN int `json:"keyN"` // number of key/value pairs } diff --git a/cmd/bbolt/main.go b/cmd/bbolt/main.go index a9256a699..e0b7635f5 100644 --- a/cmd/bbolt/main.go +++ b/cmd/bbolt/main.go @@ -370,10 +370,10 @@ func newPageItemCommand(m *Main) *pageItemCommand { } type pageItemOptions struct { + format string help bool keyOnly bool valueOnly bool - format string } // Run executes the command. @@ -1617,8 +1617,8 @@ func (r *BenchResults) OpsPerSecond() int { } type PageError struct { - ID int Err error + ID int } func (e *PageError) Error() string { diff --git a/db.go b/db.go index 236698212..17d6b3c4e 100644 --- a/db.go +++ b/db.go @@ -41,28 +41,23 @@ type DB struct { // refer to discussion in https://github.com/etcd-io/bbolt/issues/577. stats Stats - // When enabled, the database will perform a Check() after every commit. - // A panic is issued if the database is in an inconsistent state. This - // flag has a large performance impact so it should only be used for - // debugging purposes. - StrictMode bool + pagePool sync.Pool - // Setting the NoSync flag will cause the database to skip fsync() - // calls after each commit. This can be useful when bulk loading data - // into a database and you can restart the bulk load in the event of - // a system failure or database corruption. Do not set this flag for - // normal use. - // - // If the package global IgnoreNoSync constant is true, this value is - // ignored. See the comment on that constant for more details. - // - // THIS IS UNSAFE. PLEASE USE WITH CAUTION. - NoSync bool + logger Logger - // When true, skips syncing freelist to disk. This improves the database - // write performance under normal operation, but requires a full database - // re-sync during recovery. - NoFreelistSync bool + openFile func(string, int, os.FileMode) (*os.File, error) + file *os.File + data *[maxMapSize]byte + meta0 *common.Meta + meta1 *common.Meta + rwtx *Tx + + freelist *freelist + batch *batch + + ops struct { + writeAt func(b []byte, off int64) (n int, err error) + } // FreelistType sets the backend freelist type. There are two options. Array which is simple but endures // dramatic performance degradation if database is large and fragmentation in freelist is common. @@ -71,18 +66,12 @@ type DB struct { // The default type is array FreelistType FreelistType - // When true, skips the truncate call when growing the database. - // Setting this to true is only safe on non-ext3/ext4 systems. - // Skipping truncation avoids preallocation of hard drive space and - // bypasses a truncate() and fsync() syscall on remapping. - // - // https://github.com/boltdb/bolt/issues/284 - NoGrowSync bool - - // When `true`, bbolt will always load the free pages when opening the DB. - // When opening db in write mode, this flag will always automatically - // set to `true`. - PreLoadFreelist bool + path string + // `dataref` isn't used at all on Windows, and the golangci-lint + // always fails on Windows platform. + //nolint + dataref []byte // mmap'ed readonly, write throws SEGV + txs []*Tx // If you want to read the entire database fast, you can set MmapFlag to // syscall.MAP_POPULATE on Linux 2.6.23+ for sequential read-ahead. @@ -109,46 +98,61 @@ type DB struct { // of truncate() and fsync() when growing the data file. AllocSize int - // Mlock locks database file in memory when set to true. - // It prevents major page faults, however used memory can't be reclaimed. - // - // Supported only on Unix via mlock/munlock syscalls. - Mlock bool - - logger Logger - - path string - openFile func(string, int, os.FileMode) (*os.File, error) - file *os.File - // `dataref` isn't used at all on Windows, and the golangci-lint - // always fails on Windows platform. - //nolint - dataref []byte // mmap'ed readonly, write throws SEGV - data *[maxMapSize]byte datasz int - meta0 *common.Meta - meta1 *common.Meta pageSize int - opened bool - rwtx *Tx - txs []*Tx + mmaplock sync.RWMutex // Protects mmap access during remapping. + statlock sync.RWMutex // Protects stats access. - freelist *freelist freelistLoad sync.Once - pagePool sync.Pool - batchMu sync.Mutex - batch *batch - rwlock sync.Mutex // Allows only one writer at a time. - metalock sync.Mutex // Protects meta page access. - mmaplock sync.RWMutex // Protects mmap access during remapping. - statlock sync.RWMutex // Protects stats access. + rwlock sync.Mutex // Allows only one writer at a time. + metalock sync.Mutex // Protects meta page access. - ops struct { - writeAt func(b []byte, off int64) (n int, err error) - } + // When enabled, the database will perform a Check() after every commit. + // A panic is issued if the database is in an inconsistent state. This + // flag has a large performance impact so it should only be used for + // debugging purposes. + StrictMode bool + + // Setting the NoSync flag will cause the database to skip fsync() + // calls after each commit. This can be useful when bulk loading data + // into a database and you can restart the bulk load in the event of + // a system failure or database corruption. Do not set this flag for + // normal use. + // + // If the package global IgnoreNoSync constant is true, this value is + // ignored. See the comment on that constant for more details. + // + // THIS IS UNSAFE. PLEASE USE WITH CAUTION. + NoSync bool + + // When true, skips syncing freelist to disk. This improves the database + // write performance under normal operation, but requires a full database + // re-sync during recovery. + NoFreelistSync bool + + // When true, skips the truncate call when growing the database. + // Setting this to true is only safe on non-ext3/ext4 systems. + // Skipping truncation avoids preallocation of hard drive space and + // bypasses a truncate() and fsync() syscall on remapping. + // + // https://github.com/boltdb/bolt/issues/284 + NoGrowSync bool + + // When `true`, bbolt will always load the free pages when opening the DB. + // When opening db in write mode, this flag will always automatically + // set to `true`. + PreLoadFreelist bool + + // Mlock locks database file in memory when set to true. + // It prevents major page faults, however used memory can't be reclaimed. + // + // Supported only on Unix via mlock/munlock syscalls. + Mlock bool + + opened bool // Read only mode. // When true, Update() and Begin(true) return ErrDatabaseReadOnly immediately. @@ -995,8 +999,8 @@ type call struct { type batch struct { db *DB timer *time.Timer - start sync.Once calls []call + start sync.Once } // trigger runs the batch if it hasn't already been run. @@ -1263,21 +1267,13 @@ func (db *DB) freepages() []common.Pgid { // Options represents the options that can be set when opening a database. type Options struct { - // Timeout is the amount of time to wait to obtain a file lock. - // When set to zero it will wait indefinitely. - Timeout time.Duration - - // Sets the DB.NoGrowSync flag before memory mapping the file. - NoGrowSync bool - // Do not sync freelist to disk. This improves the database write performance - // under normal operation, but requires a full database re-sync during recovery. - NoFreelistSync bool + // Logger is the logger used for bbolt. + Logger Logger - // PreLoadFreelist sets whether to load the free pages when opening - // the db file. Note when opening db in write mode, bbolt will always - // load the free pages. - PreLoadFreelist bool + // OpenFile is used to open files. It defaults to os.OpenFile. This option + // is useful for writing hermetic tests. + OpenFile func(string, int, os.FileMode) (*os.File, error) // FreelistType sets the backend freelist type. There are two options. Array which is simple but endures // dramatic performance degradation if database is large and fragmentation in freelist is common. @@ -1286,9 +1282,9 @@ type Options struct { // The default type is array FreelistType FreelistType - // Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to - // grab a shared lock (UNIX). - ReadOnly bool + // Timeout is the amount of time to wait to obtain a file lock. + // When set to zero it will wait indefinitely. + Timeout time.Duration // Sets the DB.MmapFlags flag before memory mapping the file. MmapFlags int @@ -1306,22 +1302,31 @@ type Options struct { // PageSize overrides the default OS page size. PageSize int + // Sets the DB.NoGrowSync flag before memory mapping the file. + NoGrowSync bool + + // Do not sync freelist to disk. This improves the database write performance + // under normal operation, but requires a full database re-sync during recovery. + NoFreelistSync bool + + // PreLoadFreelist sets whether to load the free pages when opening + // the db file. Note when opening db in write mode, bbolt will always + // load the free pages. + PreLoadFreelist bool + + // Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to + // grab a shared lock (UNIX). + ReadOnly bool + // NoSync sets the initial value of DB.NoSync. Normally this can just be // set directly on the DB itself when returned from Open(), but this option // is useful in APIs which expose Options but not the underlying DB. NoSync bool - // OpenFile is used to open files. It defaults to os.OpenFile. This option - // is useful for writing hermetic tests. - OpenFile func(string, int, os.FileMode) (*os.File, error) - // Mlock locks database file in memory when set to true. // It prevents potential page faults, however // used memory can't be reclaimed. (UNIX only) Mlock bool - - // Logger is the logger used for bbolt. - Logger Logger } func (o *Options) String() string { diff --git a/freelist.go b/freelist.go index 49b6c200b..e60c026b0 100644 --- a/freelist.go +++ b/freelist.go @@ -23,9 +23,6 @@ type pidSet map[common.Pgid]struct{} // freelist represents a list of all pages that are available for allocation. // It also tracks pages that have been freed but are still in use by open transactions. type freelist struct { - freelistType FreelistType // freelist type - ids []common.Pgid // all free and available free page ids. - readonlyTXIDs []common.Txid // all readonly transaction IDs. allocs map[common.Pgid]common.Txid // mapping of Txid that allocated a pgid. pending map[common.Txid]*txPending // mapping of soon-to-be free page ids by tx. cache map[common.Pgid]struct{} // fast lookup of all free and pending page ids. @@ -38,6 +35,9 @@ type freelist struct { mergeSpans func(ids common.Pgids) // the mergeSpan func getFreePageIDs func() []common.Pgid // get free pgids func readIDs func(pgids []common.Pgid) // readIDs func reads list of pages and init the freelist + freelistType FreelistType // freelist type + ids []common.Pgid // all free and available free page ids. + readonlyTXIDs []common.Txid // all readonly transaction IDs. } // newFreelist returns an empty, initialized freelist. diff --git a/internal/btesting/btesting.go b/internal/btesting/btesting.go index c83369f09..d29a87c63 100644 --- a/internal/btesting/btesting.go +++ b/internal/btesting/btesting.go @@ -26,10 +26,10 @@ const ( // DB is a test wrapper for bolt.DB. type DB struct { + t testing.TB *bolt.DB - f string o *bolt.Options - t testing.TB + f string } // MustCreateDB returns a new, open DB at a temporary location. diff --git a/internal/common/inode.go b/internal/common/inode.go index 080b9af78..43333396b 100644 --- a/internal/common/inode.go +++ b/internal/common/inode.go @@ -6,10 +6,10 @@ import "unsafe" // It can be used to point to elements in a page or point // to an element which hasn't been added to a page yet. type Inode struct { - flags uint32 - pgid Pgid key []byte value []byte + pgid Pgid + flags uint32 } type Inodes []Inode diff --git a/internal/common/page.go b/internal/common/page.go index ee808967c..7a67f4c26 100644 --- a/internal/common/page.go +++ b/internal/common/page.go @@ -322,8 +322,8 @@ func (n *leafPageElement) Bucket() *InBucket { // PageInfo represents human readable information about a page. type PageInfo struct { - ID int Type string + ID int Count int OverflowCount int } diff --git a/node.go b/node.go index fe67c3c89..ec7675a7b 100644 --- a/node.go +++ b/node.go @@ -11,14 +11,14 @@ import ( // node represents an in-memory, deserialized page. type node struct { bucket *Bucket - isLeaf bool - unbalanced bool - spilled bool - key []byte - pgid common.Pgid parent *node + key []byte children nodes inodes common.Inodes + pgid common.Pgid + isLeaf bool + unbalanced bool + spilled bool } // root returns the top-level node this node is attached to.