From 698a5c4072dc6d8905e96e79af68a30a7d0b2458 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 28 Jan 2024 00:19:24 -0500 Subject: [PATCH] feat: make paging behavior more consistent, add tests (#70) --- pkg/btree/multi.go | 84 +++++++++++++++++++++---- pkg/btree/multi_test.go | 134 +++++++++++++++++++++++++++++++++++++--- pkg/btree/pagefile.go | 3 +- 3 files changed, 200 insertions(+), 21 deletions(-) diff --git a/pkg/btree/multi.go b/pkg/btree/multi.go index cdc00db7..0e98a93c 100644 --- a/pkg/btree/multi.go +++ b/pkg/btree/multi.go @@ -3,9 +3,20 @@ package btree import ( "encoding/binary" "errors" + "fmt" "io" ) +/** + * LinkedMetaPage is a linked list of meta pages. Each page contains + * a pointer to the root of the B+ tree, a pointer to the next meta page, + * and the remainder of the page is allocated as free space for metadata. + * + * A page exists if and only if the offset is not math.MaxUint64 and the + * read/write/seek pager can read one full page at the offset. The last + * page in the linked list will have a next pointer with offset + * math.MaxUint64. + */ type LinkedMetaPage struct { rws ReadWriteSeekPager offset uint64 @@ -66,20 +77,16 @@ func (m *LinkedMetaPage) Next() (*LinkedMetaPage, error) { if err := binary.Read(m.rws, binary.LittleEndian, &next); err != nil { return nil, err } - if next.Offset == 0 { - return nil, nil - } return &LinkedMetaPage{rws: m.rws, offset: next.Offset}, nil } func (m *LinkedMetaPage) AddNext() (*LinkedMetaPage, error) { - // check that the next pointer is zero curr, err := m.Next() if err != nil { return nil, err } - if curr != nil { - return nil, errors.New("next pointer is not zero") + if curr.offset != ^uint64(0) { + return nil, errors.New("next pointer already exists") } offset, err := m.rws.NewPage() if err != nil { @@ -104,24 +111,75 @@ func (m *LinkedMetaPage) MemoryPointer() MemoryPointer { } func (m *LinkedMetaPage) Exists() (bool, error) { - length, err := m.rws.Seek(0, io.SeekEnd) - if err != nil { + if m.offset == ^uint64(0) { + return false, nil + } + // attempt to read the page + if _, err := m.rws.Seek(int64(m.offset), io.SeekStart); err != nil { return false, err } - return length > int64(m.offset), nil + if _, err := m.rws.Read(make([]byte, m.rws.PageSize())); err != nil { + if err == io.EOF { + return false, nil + } + return false, err + } + return true, nil } func (m *LinkedMetaPage) Reset() error { + // write a full page of zeros + emptyPage := make([]byte, m.rws.PageSize()) + binary.BigEndian.PutUint64(emptyPage[12:20], ^uint64(0)) if _, err := m.rws.Seek(int64(m.offset), io.SeekStart); err != nil { return err } - // write 28 bytes of zeros - if _, err := m.rws.Write(make([]byte, 28)); err != nil { + if _, err := m.rws.Write(emptyPage); err != nil { return err } return nil } -func NewMultiBPTree(t ReadWriteSeekPager, offset uint64) *LinkedMetaPage { - return &LinkedMetaPage{rws: t, offset: offset} +// Collect returns a slice of all linked meta pages from this page to the end. +// This function is useful for debugging and testing, however generally it should +// not be used for functional code. +func (m *LinkedMetaPage) Collect() ([]*LinkedMetaPage, error) { + var pages []*LinkedMetaPage + node := m + for { + exists, err := node.Exists() + if err != nil { + return nil, err + } + if !exists { + break + } + pages = append(pages, node) + next, err := node.Next() + if err != nil { + return nil, err + } + node = next + } + return pages, nil +} + +func (m *LinkedMetaPage) String() string { + nm, err := m.Next() + if err != nil { + panic(err) + } + root, err := m.Root() + if err != nil { + panic(err) + } + return fmt.Sprintf("LinkedMetaPage{offset: %x,\tnext: %x,\troot: %x}", m.offset, nm.offset, root.Offset) +} + +func NewMultiBPTree(t ReadWriteSeekPager, page int) (*LinkedMetaPage, error) { + offset, err := t.Page(0) + if err != nil { + return nil, err + } + return &LinkedMetaPage{rws: t, offset: uint64(offset)}, nil } diff --git a/pkg/btree/multi_test.go b/pkg/btree/multi_test.go index f91cea78..4695f39d 100644 --- a/pkg/btree/multi_test.go +++ b/pkg/btree/multi_test.go @@ -14,13 +14,16 @@ func TestMultiBPTree(t *testing.T) { if err != nil { t.Fatal(err) } - tree := NewMultiBPTree(p, uint64(p.PageSize())) + tree, err := NewMultiBPTree(p, 0) + if err != nil { + t.Fatal(err) + } exists, err := tree.Exists() if err != nil { t.Fatal(err) } if exists { - t.Fatal("expected not found") + t.Fatalf("expected not found, got page %v", tree) } }) @@ -30,7 +33,10 @@ func TestMultiBPTree(t *testing.T) { if err != nil { t.Fatal(err) } - tree := NewMultiBPTree(p, uint64(p.PageSize())) + tree, err := NewMultiBPTree(p, 0) + if err != nil { + t.Fatal(err) + } if err := tree.Reset(); err != nil { t.Fatal(err) } @@ -53,7 +59,10 @@ func TestMultiBPTree(t *testing.T) { if err != nil { t.Fatal(err) } - tree := NewMultiBPTree(p, uint64(p.PageSize())) + tree, err := NewMultiBPTree(p, 0) + if err != nil { + t.Fatal(err) + } if err := tree.Reset(); err != nil { t.Fatal(err) } @@ -92,7 +101,10 @@ func TestMultiBPTree(t *testing.T) { if err != nil { t.Fatal(err) } - tree := NewMultiBPTree(p, uint64(p.PageSize())) + tree, err := NewMultiBPTree(p, 0) + if err != nil { + t.Fatal(err) + } if err := tree.Reset(); err != nil { t.Fatal(err) } @@ -115,7 +127,10 @@ func TestMultiBPTree(t *testing.T) { if err != nil { t.Fatal(err) } - tree := NewMultiBPTree(p, uint64(p.PageSize())) + tree, err := NewMultiBPTree(p, 0) + if err != nil { + t.Fatal(err) + } if err := tree.Reset(); err != nil { t.Fatal(err) } @@ -134,7 +149,10 @@ func TestMultiBPTree(t *testing.T) { if err != nil { t.Fatal(err) } - tree := NewMultiBPTree(p, uint64(p.PageSize())) + tree, err := NewMultiBPTree(p, 0) + if err != nil { + t.Fatal(err) + } if err := tree.Reset(); err != nil { t.Fatal(err) } @@ -149,4 +167,106 @@ func TestMultiBPTree(t *testing.T) { t.Fatalf("got %v want %v", metadata, []byte("hello")) } }) + + t.Run("setting metadata too large fails", func(t *testing.T) { + b := buftest.NewSeekableBuffer() + p, err := NewPageFile(b) + if err != nil { + t.Fatal(err) + } + tree, err := NewMultiBPTree(p, 0) + if err != nil { + t.Fatal(err) + } + if err := tree.Reset(); err != nil { + t.Fatal(err) + } + if err := tree.SetMetadata(make([]byte, 4096)); err == nil { + t.Fatal("expected error") + } + }) + + t.Run("collect pages", func(t *testing.T) { + b := buftest.NewSeekableBuffer() + p, err := NewPageFile(b) + if err != nil { + t.Fatal(err) + } + tree, err := NewMultiBPTree(p, 0) + if err != nil { + t.Fatal(err) + } + if err := tree.Reset(); err != nil { + t.Fatal(err) + } + + // Create a linked list of LinkedMetaPages + page1, err := tree.AddNext() + if err != nil { + t.Fatal(err) + } + page2, err := page1.AddNext() + if err != nil { + t.Fatal(err) + } + page3, err := page2.AddNext() + if err != nil { + t.Fatal(err) + } + + // Collect the pages + collectedPages, err := page1.Collect() + if err != nil { + t.Fatal(err) + } + + // Verify the collected pages + expectedPages := []*LinkedMetaPage{page1, page2, page3} + if !reflect.DeepEqual(collectedPages, expectedPages) { + t.Fatalf("got %v, want %v", collectedPages, expectedPages) + } + }) + + t.Run("singular list", func(t *testing.T) { + b := buftest.NewSeekableBuffer() + p, err := NewPageFile(b) + if err != nil { + t.Fatal(err) + } + tree, err := NewMultiBPTree(p, 0) + if err != nil { + t.Fatal(err) + } + if err := tree.Reset(); err != nil { + t.Fatal(err) + } + collectedPages, err := tree.Collect() + if err != nil { + t.Fatal(err) + } + expectedPages := []*LinkedMetaPage{tree} + if !reflect.DeepEqual(collectedPages, expectedPages) { + t.Fatalf("got %v, want %v", collectedPages, expectedPages) + } + }) + + t.Run("empty list", func(t *testing.T) { + b := buftest.NewSeekableBuffer() + p, err := NewPageFile(b) + if err != nil { + t.Fatal(err) + } + + tree, err := NewMultiBPTree(p, 0) + if err != nil { + t.Fatal(err) + } + collectedPages, err := tree.Collect() + if err != nil { + t.Fatal(err) + } + if collectedPages != nil { + t.Fatalf("got %v, want nil", collectedPages) + } + }) } diff --git a/pkg/btree/pagefile.go b/pkg/btree/pagefile.go index 906b4e4a..fe316dc9 100644 --- a/pkg/btree/pagefile.go +++ b/pkg/btree/pagefile.go @@ -61,7 +61,8 @@ func (pf *PageFile) Page(i int) (int64, error) { if i < 0 { return 0, errors.New("page index cannot be negative") } - return int64(i) * int64(pf.pageSize), nil + // i + 1 because the first page is reserved for the free page indexes + return int64(i+1) * int64(pf.pageSize), nil } func (pf *PageFile) NewPage() (int64, error) {