diff --git a/pkg/btree/bptree.go b/pkg/btree/bptree.go index f6b037b4..4f26152e 100644 --- a/pkg/btree/bptree.go +++ b/pkg/btree/bptree.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "io" - "slices" ) // MetaPage is an abstract interface over the root page of a btree @@ -15,20 +14,15 @@ type MetaPage interface { SetRoot(MemoryPointer) error } -type ReadWriteSeekTruncater interface { - io.ReadWriteSeeker - Truncate(size int64) error -} - type BPTree struct { - tree ReadWriteSeekTruncater + tree ReadWriteSeekPager meta MetaPage maxPageSize int } -func NewBPTree(tree ReadWriteSeekTruncater, meta MetaPage, maxPageSize int) *BPTree { - return &BPTree{tree: tree, meta: meta, maxPageSize: maxPageSize} +func NewBPTree(tree ReadWriteSeekPager, meta MetaPage) *BPTree { + return &BPTree{tree: tree, meta: meta} } func (t *BPTree) root() (*BPTreeNode, MemoryPointer, error) { @@ -116,7 +110,7 @@ func (t *BPTree) Insert(key ReferencedValue, value MemoryPointer) error { } if root == nil { // special case, create the root as the first node - offset, err := t.tree.Seek(0, io.SeekEnd) + offset, err := t.tree.NewPage() if err != nil { return err } @@ -151,9 +145,9 @@ func (t *BPTree) Insert(key ReferencedValue, value MemoryPointer) error { for i := 0; i < len(path); i++ { tr := path[i] n := tr.node - if len(n.Keys) > t.maxPageSize { + if len(n.Keys) > t.tree.PageSize() { // split the node - moffset, err := t.tree.Seek(0, io.SeekEnd) + moffset, err := t.tree.NewPage() if err != nil { return err } @@ -184,7 +178,11 @@ func (t *BPTree) Insert(key ReferencedValue, value MemoryPointer) error { n.Pointers = n.Pointers[:mid+1] n.Keys = n.Keys[:mid] } - noffset := moffset + msize + + noffset, err := t.tree.NewPage() + if err != nil { + return err + } nsize, err := n.WriteTo(t.tree) if err != nil { return err @@ -227,7 +225,7 @@ func (t *BPTree) Insert(key ReferencedValue, value MemoryPointer) error { } } else { // write this node to disk and update the parent - offset, err := t.tree.Seek(0, io.SeekEnd) + offset, err := t.tree.NewPage() if err != nil { return err } @@ -337,174 +335,3 @@ type Entry struct { // } // } // } - -func (t *BPTree) compact() error { - // read all the nodes and compile a list of nodes still referenced, - // then write out the nodes in order, removing unreferenced nodes and updating - // the parent pointers. - - _, rootOffset, err := t.root() - if err != nil { - return err - } - - if _, err := t.tree.Seek(0, io.SeekStart); err != nil { - return err - } - - references := []MemoryPointer{rootOffset} - for { - node := &BPTreeNode{} - if _, err := node.ReadFrom(t.tree); err != nil { - if err == io.EOF { - break - } - return err - } - if !node.leaf() { - // all pointers are references - references = append(references, node.Pointers...) - } - } - - // read all the nodes again and write out the referenced nodes - if _, err := t.tree.Seek(0, io.SeekStart); err != nil { - return err - } - - slices.SortFunc(references, func(x, y MemoryPointer) int { - return int(x.Offset - y.Offset) - }) - - referenceMap := make(map[uint64]MemoryPointer) - - offset := 0 - for i, reference := range references { - // skip duplicates - if i > 0 && references[i-1] == reference { - continue - } - // read the referenced node - if _, err := t.tree.Seek(int64(reference.Offset), io.SeekStart); err != nil { - return err - } - node := &BPTreeNode{} - if _, err := node.ReadFrom(t.tree); err != nil { - return err - } - // write the node to the new offset - if _, err := t.tree.Seek(int64(offset), io.SeekStart); err != nil { - return err - } - n, err := node.WriteTo(t.tree) - if err != nil { - return err - } - // update the reference map - referenceMap[reference.Offset] = MemoryPointer{Offset: uint64(offset), Length: uint32(n)} - offset += int(n) - } - - // truncate the file - if err := t.tree.Truncate(int64(offset)); err != nil { - return err - } - - // update the parent pointers - if _, err := t.tree.Seek(0, io.SeekStart); err != nil { - return err - } - for { - offset, err := t.tree.Seek(0, io.SeekCurrent) - if err != nil { - return err - } - node := &BPTreeNode{} - if _, err := node.ReadFrom(t.tree); err != nil { - if err == io.EOF { - break - } - return err - } - if !node.leaf() { - // all pointers are references - for i, p := range node.Pointers { - node.Pointers[i] = referenceMap[p.Offset] - } - } - if _, err := t.tree.Seek(offset, io.SeekStart); err != nil { - return err - } - if _, err := node.WriteTo(t.tree); err != nil { - return err - } - } - - // update the meta pointer - return t.meta.SetRoot(referenceMap[rootOffset.Offset]) -} - -func (t *BPTree) String() string { - var buf bytes.Buffer - // get the current seek position - seekPos, err := t.tree.Seek(0, io.SeekCurrent) - if err != nil { - return err.Error() - } - defer func() { - // reset the seek position - if _, err := t.tree.Seek(seekPos, io.SeekStart); err != nil { - panic(err) - } - }() - root, rootOffset, err := t.root() - if err != nil { - return err.Error() - } - if root == nil { - return "empty tree" - } - if _, err := buf.Write([]byte(fmt.Sprintf("root: %d\n", rootOffset))); err != nil { - return err.Error() - } - // seek to 8 - if _, err := t.tree.Seek(0, io.SeekStart); err != nil { - return err.Error() - } - for { - offset, err := t.tree.Seek(0, io.SeekCurrent) - if err != nil { - return err.Error() - } - node := &BPTreeNode{} - if _, err := node.ReadFrom(t.tree); err != nil { - if err == io.EOF { - break - } - return err.Error() - } - if node.leaf() { - if _, err := buf.Write([]byte(fmt.Sprintf("%04d | ", offset))); err != nil { - return err.Error() - } - } else { - if _, err := buf.Write([]byte(fmt.Sprintf("%04d ", offset))); err != nil { - return err.Error() - } - } - for i := 0; i < len(node.Pointers); i++ { - if _, err := buf.Write([]byte(fmt.Sprintf("%04d ", node.Pointers[i]))); err != nil { - return err.Error() - } - if i < len(node.Keys) { - if _, err := buf.Write([]byte(fmt.Sprintf("%02d ", node.Keys[i]))); err != nil { - return err.Error() - } - } - } - if _, err := buf.Write([]byte("\n")); err != nil { - return err.Error() - } - } - return buf.String() -} diff --git a/pkg/btree/bptree_test.go b/pkg/btree/bptree_test.go index 04edbfd3..ef1e6cd2 100644 --- a/pkg/btree/bptree_test.go +++ b/pkg/btree/bptree_test.go @@ -2,7 +2,6 @@ package btree import ( "encoding/binary" - "fmt" "testing" ) @@ -22,7 +21,11 @@ func (m *testMetaPage) Root() (MemoryPointer, error) { func TestBPTree(t *testing.T) { t.Run("empty tree", func(t *testing.T) { b := newSeekableBuffer() - tree := NewBPTree(b, &testMetaPage{}, 4096) + p, err := NewPageFile(b) + if err != nil { + t.Fatal(err) + } + tree := NewBPTree(p, &testMetaPage{}) // find a key that doesn't exist _, found, err := tree.Find([]byte("hello")) if err != nil { @@ -35,7 +38,11 @@ func TestBPTree(t *testing.T) { t.Run("insert creates a root", func(t *testing.T) { b := newSeekableBuffer() - tree := NewBPTree(b, &testMetaPage{}, 4096) + p, err := NewPageFile(b) + if err != nil { + t.Fatal(err) + } + tree := NewBPTree(p, &testMetaPage{}) if err := tree.Insert(ReferencedValue{Value: []byte("hello")}, MemoryPointer{Offset: 1}); err != nil { t.Fatal(err) } @@ -53,7 +60,11 @@ func TestBPTree(t *testing.T) { t.Run("insert into root", func(t *testing.T) { b := newSeekableBuffer() - tree := NewBPTree(b, &testMetaPage{}, 4096) + p, err := NewPageFile(b) + if err != nil { + t.Fatal(err) + } + tree := NewBPTree(p, &testMetaPage{}) if err := tree.Insert(ReferencedValue{Value: []byte("hello")}, MemoryPointer{Offset: 1}); err != nil { t.Fatal(err) } @@ -82,33 +93,13 @@ func TestBPTree(t *testing.T) { } }) - t.Run("compacting after second root insertion removes old root", func(t *testing.T) { + t.Run("split root", func(t *testing.T) { b := newSeekableBuffer() - tree := NewBPTree(b, &testMetaPage{}, 4096) - if err := tree.Insert(ReferencedValue{Value: []byte("hello")}, MemoryPointer{Offset: 1}); err != nil { - t.Fatal(err) - } - if err := tree.Insert(ReferencedValue{Value: []byte("world")}, MemoryPointer{Offset: 2}); err != nil { - t.Fatal(err) - } - if err := tree.compact(); err != nil { - t.Fatal(err) - } - v, found, err := tree.Find([]byte("world")) + p, err := NewPageFile(b) if err != nil { t.Fatal(err) } - if !found { - t.Fatal("expected to find key") - } - if v.Offset != 2 { - t.Fatalf("expected value 2, got %d", v) - } - }) - - t.Run("split root", func(t *testing.T) { - b := newSeekableBuffer() - tree := NewBPTree(b, &testMetaPage{}, 4096) + tree := NewBPTree(p, &testMetaPage{}) if err := tree.Insert(ReferencedValue{Value: []byte("hello")}, MemoryPointer{Offset: 1}); err != nil { t.Fatal(err) } @@ -121,9 +112,6 @@ func TestBPTree(t *testing.T) { if err := tree.Insert(ReferencedValue{Value: []byte("cooow")}, MemoryPointer{Offset: 4}); err != nil { t.Fatal(err) } - if err := tree.compact(); err != nil { - t.Fatal(err) - } v1, f1, err := tree.Find([]byte("hello")) if err != nil { t.Fatal(err) @@ -168,47 +156,45 @@ func TestBPTree(t *testing.T) { t.Run("split intermediate", func(t *testing.T) { b := newSeekableBuffer() - tree := NewBPTree(b, &testMetaPage{}, 2) + p, err := NewPageFile(b) + if err != nil { + t.Fatal(err) + } + tree := NewBPTree(p, &testMetaPage{}) if err := tree.Insert(ReferencedValue{Value: []byte{0x05}}, MemoryPointer{Offset: 5}); err != nil { t.Fatal(err) } - fmt.Printf("inserted a\n") - fmt.Printf(tree.String()) if err := tree.Insert(ReferencedValue{Value: []byte{0x15}}, MemoryPointer{Offset: 15}); err != nil { t.Fatal(err) } - fmt.Printf("inserted b\n") - fmt.Printf(tree.String()) if err := tree.Insert(ReferencedValue{Value: []byte{0x25}}, MemoryPointer{Offset: 25}); err != nil { t.Fatal(err) } - fmt.Printf("inserted c\n") - fmt.Printf(tree.String()) if err := tree.Insert(ReferencedValue{Value: []byte{0x35}}, MemoryPointer{Offset: 35}); err != nil { t.Fatal(err) } - fmt.Printf("inserted d\n") - fmt.Printf(tree.String()) if err := tree.Insert(ReferencedValue{Value: []byte{0x45}}, MemoryPointer{Offset: 45}); err != nil { t.Fatal(err) } - fmt.Printf("inserted e\n") - fmt.Printf(tree.String()) }) t.Run("insertion test", func(t *testing.T) { b := newSeekableBuffer() - tree := NewBPTree(b, &testMetaPage{}, 512) - for i := 0; i < 10240; i++ { - buf := make([]byte, 4) - binary.BigEndian.PutUint32(buf, uint32(i)) + p, err := NewPageFile(b) + if err != nil { + t.Fatal(err) + } + tree := NewBPTree(p, &testMetaPage{}) + for i := 0; i < 16384; i++ { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(i)) if err := tree.Insert(ReferencedValue{Value: buf}, MemoryPointer{Offset: uint64(i)}); err != nil { t.Fatal(err) } } - for i := 0; i < 10240; i++ { - buf := make([]byte, 4) - binary.BigEndian.PutUint32(buf, uint32(i)) + for i := 0; i < 16384; i++ { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(i)) v, found, err := tree.Find(buf) if err != nil { t.Fatal(err) diff --git a/pkg/btree/multi.go b/pkg/btree/multi.go index 6a646c98..90024c65 100644 --- a/pkg/btree/multi.go +++ b/pkg/btree/multi.go @@ -7,12 +7,12 @@ import ( ) type LinkedMetaPage struct { - rws io.ReadWriteSeeker + rws ReadWriteSeekPager offset uint64 } func (m *LinkedMetaPage) Root() (MemoryPointer, error) { - if _, err := m.rws.Seek(int64(m.offset), 0); err != nil { + if _, err := m.rws.Seek(int64(m.offset), io.SeekStart); err != nil { return MemoryPointer{}, err } var mp MemoryPointer @@ -20,14 +20,14 @@ func (m *LinkedMetaPage) Root() (MemoryPointer, error) { } func (m *LinkedMetaPage) SetRoot(mp MemoryPointer) error { - if _, err := m.rws.Seek(int64(m.offset), 0); err != nil { + if _, err := m.rws.Seek(int64(m.offset), io.SeekStart); err != nil { return err } return binary.Write(m.rws, binary.LittleEndian, mp) } func (m *LinkedMetaPage) Metadata() (MemoryPointer, error) { - if _, err := m.rws.Seek(int64(m.offset)+12, 0); err != nil { + if _, err := m.rws.Seek(int64(m.offset)+12, io.SeekStart); err != nil { return MemoryPointer{}, err } var mp MemoryPointer @@ -35,14 +35,14 @@ func (m *LinkedMetaPage) Metadata() (MemoryPointer, error) { } func (m *LinkedMetaPage) SetMetadata(mp MemoryPointer) error { - if _, err := m.rws.Seek(int64(m.offset)+12, 0); err != nil { + if _, err := m.rws.Seek(int64(m.offset)+12, io.SeekStart); err != nil { return err } return binary.Write(m.rws, binary.LittleEndian, mp) } func (m *LinkedMetaPage) Next() (*LinkedMetaPage, error) { - if _, err := m.rws.Seek(int64(m.offset)+24, 0); err != nil { + if _, err := m.rws.Seek(int64(m.offset)+24, io.SeekStart); err != nil { return nil, err } var next MemoryPointer @@ -64,7 +64,7 @@ func (m *LinkedMetaPage) AddNext() (*LinkedMetaPage, error) { if curr != nil { return nil, errors.New("next pointer is not zero") } - offset, err := m.rws.Seek(0, io.SeekEnd) + offset, err := m.rws.NewPage() if err != nil { return nil, err } @@ -73,7 +73,7 @@ func (m *LinkedMetaPage) AddNext() (*LinkedMetaPage, error) { return nil, err } // save the next pointer - if _, err := m.rws.Seek(int64(m.offset)+24, 0); err != nil { + if _, err := m.rws.Seek(int64(m.offset)+24, io.SeekStart); err != nil { return nil, err } if err := binary.Write(m.rws, binary.LittleEndian, next.offset); err != nil { @@ -105,6 +105,10 @@ func (m *LinkedMetaPage) Reset() error { return nil } -func NewMultiBPTree(t ReadWriteSeekTruncater) *LinkedMetaPage { - return &LinkedMetaPage{rws: t, offset: 0} +func NewMultiBPTree(t ReadWriteSeekPager) (*LinkedMetaPage, error) { + offset, err := t.NewPage() + if err != nil { + return nil, err + } + return &LinkedMetaPage{rws: t, offset: uint64(offset)}, nil } diff --git a/pkg/btree/multi_test.go b/pkg/btree/multi_test.go index 82641879..cca2228b 100644 --- a/pkg/btree/multi_test.go +++ b/pkg/btree/multi_test.go @@ -8,7 +8,14 @@ import ( func TestMultiBPTree(t *testing.T) { t.Run("empty tree", func(t *testing.T) { b := newSeekableBuffer() - tree := NewMultiBPTree(b) + p, err := NewPageFile(b) + if err != nil { + t.Fatal(err) + } + tree, err := NewMultiBPTree(p) + if err != nil { + t.Fatal(err) + } exists, err := tree.Exists() if err != nil { t.Fatal(err) @@ -20,7 +27,14 @@ func TestMultiBPTree(t *testing.T) { t.Run("reset tree", func(t *testing.T) { b := newSeekableBuffer() - tree := NewMultiBPTree(b) + p, err := NewPageFile(b) + if err != nil { + t.Fatal(err) + } + tree, err := NewMultiBPTree(p) + if err != nil { + t.Fatal(err) + } if err := tree.Reset(); err != nil { t.Fatal(err) } @@ -32,9 +46,6 @@ func TestMultiBPTree(t *testing.T) { t.Fatal("expected found") } mp := tree.MemoryPointer() - if mp.Offset != 0 { - t.Fatalf("expected offset 0, got %d", mp.Offset) - } if mp.Length != 36 { t.Fatalf("expected length 36, got %d", mp.Length) } @@ -42,7 +53,14 @@ func TestMultiBPTree(t *testing.T) { t.Run("insert a second page", func(t *testing.T) { b := newSeekableBuffer() - tree := NewMultiBPTree(b) + p, err := NewPageFile(b) + if err != nil { + t.Fatal(err) + } + tree, err := NewMultiBPTree(p) + if err != nil { + t.Fatal(err) + } if err := tree.Reset(); err != nil { t.Fatal(err) } @@ -50,9 +68,6 @@ func TestMultiBPTree(t *testing.T) { if err != nil { t.Fatal(err) } - if next1.MemoryPointer().Offset != 36 { - t.Fatalf("expected offset 36, got %d", next1) - } if next1.MemoryPointer().Length != 36 { t.Fatalf("expected length 36, got %d", next1) } @@ -60,13 +75,14 @@ func TestMultiBPTree(t *testing.T) { if err != nil { t.Fatal(err) } - if next2.MemoryPointer().Offset != 72 { - t.Fatalf("expected offset 72, got %d", next2) - } if next2.MemoryPointer().Length != 36 { t.Fatalf("expected length 36, got %d", next2) } + if next1.MemoryPointer().Offset == next2.MemoryPointer().Offset { + t.Fatalf("expected different offsets, got %d", next1.MemoryPointer().Offset) + } + // check the first page m1, err := tree.Next() if err != nil { @@ -79,7 +95,14 @@ func TestMultiBPTree(t *testing.T) { t.Run("duplicate next pointer", func(t *testing.T) { b := newSeekableBuffer() - tree := NewMultiBPTree(b) + p, err := NewPageFile(b) + if err != nil { + t.Fatal(err) + } + tree, err := NewMultiBPTree(p) + if err != nil { + t.Fatal(err) + } if err := tree.Reset(); err != nil { t.Fatal(err) } @@ -87,9 +110,6 @@ func TestMultiBPTree(t *testing.T) { if err != nil { t.Fatal(err) } - if next1.MemoryPointer().Offset != 36 { - t.Fatalf("expected offset 36, got %d", next1) - } if next1.MemoryPointer().Length != 36 { t.Fatalf("expected length 36, got %d", next1) } diff --git a/pkg/btree/pagefile.go b/pkg/btree/pagefile.go index 271b4469..a1ff316b 100644 --- a/pkg/btree/pagefile.go +++ b/pkg/btree/pagefile.go @@ -7,16 +7,18 @@ import ( "log" ) -type ReadWritePager interface { - io.ReadWriter +type ReadWriteSeekPager interface { + io.ReadWriteSeeker NewPage() (int64, error) FreePage(int64) error + + PageSize() int } type PageFile struct { io.ReadWriteSeeker - PageSize int + pageSize int // local cache of free pages to avoid reading from disk too often. freePageIndexes [512]int64 @@ -38,7 +40,7 @@ func NewPageFile(rws io.ReadWriteSeeker) (*PageFile, error) { } pf := &PageFile{ ReadWriteSeeker: rws, - PageSize: pageSizeBytes, + pageSize: pageSizeBytes, } if err == io.EOF { // allocate one page for the free page indexes @@ -81,9 +83,9 @@ func (pf *PageFile) NewPage() (int64, error) { } // if the offset is not a multiple of the page size, we need to pad the file // with zeros to the next page boundary. - if pf.PageSize > 0 && offset%int64(pf.PageSize) != 0 { + if pf.pageSize > 0 && offset%int64(pf.pageSize) != 0 { // Calculate the number of bytes to pad - pad := int64(pf.PageSize) - (offset % int64(pf.PageSize)) + pad := int64(pf.pageSize) - (offset % int64(pf.pageSize)) // Write the padding if _, err := pf.Write(make([]byte, pad)); err != nil { return 0, err @@ -94,7 +96,7 @@ func (pf *PageFile) NewPage() (int64, error) { } func (pf *PageFile) FreePage(offset int64) error { - if offset%int64(pf.PageSize) != 0 { + if offset%int64(pf.pageSize) != 0 { return errors.New("offset is not a multiple of the page size") } // find the last nonzero free page index and insert it after that @@ -114,3 +116,7 @@ func (pf *PageFile) FreePage(offset int64) error { } return errors.New("too many free pages") } + +func (pf *PageFile) PageSize() int { + return pf.pageSize +}