Skip to content

Commit

Permalink
Merge branch 'main' into refactor-btreejs
Browse files Browse the repository at this point in the history
  • Loading branch information
@friendlymatthew
friendlymatthew authored Jan 22, 2024
2 parents dd80cba + 79a8ff2 commit ea9fbc7
Show file tree
Hide file tree
Showing 9 changed files with 518 additions and 184 deletions.
449 changes: 317 additions & 132 deletions examples/client/index.html
View file

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"main": "index.js",
"scripts": {
"build": "esbuild src/index.ts --bundle --minify --sourcemap --outfile=dist/appendable.min.js",
"build-index": "go run cmd/main.go examples/workspace/green_tripdata_2023-01.jsonl",
"build-index": "go run cmd/main.go -jsonl examples/workspace/green_tripdata_2023-01.jsonl",
"serve:example": "cd examples/client && npx http-server",
"test": "jest"
},
Expand Down
46 changes: 43 additions & 3 deletions pkg/btree/bptree.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ func (t *BPTree) Insert(key ReferencedValue, value MemoryPointer) error {
for i := 0; i < len(path); i++ {
tr := path[i]
n := tr.node
if len(n.Keys) > t.tree.PageSize() {
if int(n.Size()) > t.tree.PageSize() {
// split the node
moffset, err := t.tree.NewPage()
if err != nil {
Expand Down Expand Up @@ -194,14 +194,15 @@ func (t *BPTree) Insert(key ReferencedValue, value MemoryPointer) error {
j, _ := p.node.bsearch(midKey.Value)
if j != p.index {
// j should be equal to p.index...?
// panic("aww")
// if this panic never happens then we can probably remove the above bsearch.
panic("this assumption apparently isn't true")
}
// insert the key into the parent
if j == len(p.node.Keys) {
p.node.Keys = append(p.node.Keys, midKey)
} else {
p.node.Keys = append(p.node.Keys[:j+1], p.node.Keys[j:]...)
p.node.Keys[j+1] = midKey
p.node.Keys[j] = midKey
}
p.node.Pointers = append(p.node.Pointers[:j+1], p.node.Pointers[j:]...)
p.node.Pointers[j] = MemoryPointer{Offset: uint64(noffset), Length: uint32(nsize)}
Expand Down Expand Up @@ -335,3 +336,42 @@ type Entry struct {
// }
// }
// }

func (t *BPTree) recursiveString(n *BPTreeNode, indent int) string {
// print the node itself
var buf bytes.Buffer
if !n.leaf() {
for i := range n.Pointers {
child, err := t.readNode(n.Pointers[i])
if err != nil {
return fmt.Sprintf("error: failed to read child node: %v", err)
}
buf.WriteString(t.recursiveString(child, indent+1))
if i < len(n.Pointers)-1 {
for i := 0; i < indent; i++ {
buf.WriteString(" ")
}
buf.WriteString(fmt.Sprintf("key %v\n", n.Keys[i]))
}
}
} else {
for i := range n.Pointers {
for i := 0; i < indent; i++ {
buf.WriteString(" ")
}
buf.WriteString(fmt.Sprintf("%v\n", n.Keys[i]))
}
}
return buf.String()
}

func (t *BPTree) String() string {
root, _, err := t.root()
if err != nil {
return fmt.Sprintf("error: failed to read root node: %v", err)
}
if root == nil {
return "empty tree"
}
return "b+ tree ---\n" + t.recursiveString(root, 0)
}
37 changes: 37 additions & 0 deletions pkg/btree/bptree_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package btree

import (
"encoding/binary"
"math/rand"
"testing"
)

Expand Down Expand Up @@ -208,6 +209,42 @@ func TestBPTree(t *testing.T) {
}
})

t.Run("random insertion test", func(t *testing.T) {
b := newSeekableBuffer()
p, err := NewPageFile(b)
if err != nil {
t.Fatal(err)
}
tree := NewBPTree(p, &testMetaPage{})
r := rand.New(rand.NewSource(12345))
for i := 0; i < 65536; i++ {
buf := make([]byte, 8)
if _, err := r.Read(buf); err != nil {
t.Fatal(err)
}
if err := tree.Insert(ReferencedValue{Value: buf}, MemoryPointer{Offset: uint64(i)}); err != nil {
t.Fatal(err)
}
}
s := rand.New(rand.NewSource(12345))
for i := 0; i < 65536; i++ {
buf := make([]byte, 8)
if _, err := s.Read(buf); err != nil {
t.Fatal(err)
}
v, found, err := tree.Find(buf)
if err != nil {
t.Fatal(err)
}
if !found {
t.Fatalf("expected to find key %d", i)
}
if v.Offset != uint64(i) {
t.Fatalf("expected value %d, got %d", i, v)
}
}
})

// t.Run("bulk insert", func(t *testing.T) {
// b := newSeekableBuffer()
// tree :=NewBPTree(b, 2)
Expand Down
51 changes: 32 additions & 19 deletions pkg/btree/multi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,23 +26,40 @@ func (m *LinkedMetaPage) SetRoot(mp MemoryPointer) error {
return binary.Write(m.rws, binary.LittleEndian, mp)
}

func (m *LinkedMetaPage) Metadata() (MemoryPointer, error) {
if _, err := m.rws.Seek(int64(m.offset)+12, io.SeekStart); err != nil {
return MemoryPointer{}, err
func (m *LinkedMetaPage) BPTree() *BPTree {
return NewBPTree(m.rws, m)
}

func (m *LinkedMetaPage) Metadata() ([]byte, error) {
if _, err := m.rws.Seek(int64(m.offset)+24, io.SeekStart); err != nil {
return nil, err
}
var mp MemoryPointer
return mp, binary.Read(m.rws, binary.LittleEndian, &mp)
buf := make([]byte, m.rws.PageSize()-24)
if _, err := m.rws.Read(buf); err != nil {
return nil, err
}
// the first four bytes represents the length
length := binary.LittleEndian.Uint32(buf[:4])
return buf[4 : 4+length], nil
}

func (m *LinkedMetaPage) SetMetadata(mp MemoryPointer) error {
if _, err := m.rws.Seek(int64(m.offset)+12, io.SeekStart); err != nil {
func (m *LinkedMetaPage) SetMetadata(data []byte) error {
if len(data) > m.rws.PageSize()-24 {
return errors.New("metadata too large")
}
if _, err := m.rws.Seek(int64(m.offset)+24, io.SeekStart); err != nil {
return err
}
return binary.Write(m.rws, binary.LittleEndian, mp)
buf := append(make([]byte, 4), data...)
binary.LittleEndian.PutUint32(buf, uint32(len(data)))
if _, err := m.rws.Write(buf); err != nil {
return err
}
return nil
}

func (m *LinkedMetaPage) Next() (*LinkedMetaPage, error) {
if _, err := m.rws.Seek(int64(m.offset)+24, io.SeekStart); err != nil {
if _, err := m.rws.Seek(int64(m.offset)+12, io.SeekStart); err != nil {
return nil, err
}
var next MemoryPointer
Expand Down Expand Up @@ -73,7 +90,7 @@ func (m *LinkedMetaPage) AddNext() (*LinkedMetaPage, error) {
return nil, err
}
// save the next pointer
if _, err := m.rws.Seek(int64(m.offset)+24, io.SeekStart); err != nil {
if _, err := m.rws.Seek(int64(m.offset)+12, io.SeekStart); err != nil {
return nil, err
}
if err := binary.Write(m.rws, binary.LittleEndian, next.offset); err != nil {
Expand All @@ -83,7 +100,7 @@ func (m *LinkedMetaPage) AddNext() (*LinkedMetaPage, error) {
}

func (m *LinkedMetaPage) MemoryPointer() MemoryPointer {
return MemoryPointer{Offset: m.offset, Length: 36}
return MemoryPointer{Offset: m.offset, Length: 24}
}

func (m *LinkedMetaPage) Exists() (bool, error) {
Expand All @@ -98,17 +115,13 @@ func (m *LinkedMetaPage) Reset() error {
if _, err := m.rws.Seek(int64(m.offset), io.SeekStart); err != nil {
return err
}
// write 36 bytes of zeros
if _, err := m.rws.Write(make([]byte, 36)); err != nil {
// write 28 bytes of zeros
if _, err := m.rws.Write(make([]byte, 28)); err != nil {
return err
}
return nil
}

func NewMultiBPTree(t ReadWriteSeekPager) (*LinkedMetaPage, error) {
offset, err := t.NewPage()
if err != nil {
return nil, err
}
return &LinkedMetaPage{rws: t, offset: uint64(offset)}, nil
func NewMultiBPTree(t ReadWriteSeekPager, offset uint64) *LinkedMetaPage {
return &LinkedMetaPage{rws: t, offset: offset}
}
77 changes: 53 additions & 24 deletions pkg/btree/multi_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,7 @@ func TestMultiBPTree(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tree, err := NewMultiBPTree(p)
if err != nil {
t.Fatal(err)
}
tree := NewMultiBPTree(p, uint64(p.PageSize()))
exists, err := tree.Exists()
if err != nil {
t.Fatal(err)
Expand All @@ -31,10 +28,7 @@ func TestMultiBPTree(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tree, err := NewMultiBPTree(p)
if err != nil {
t.Fatal(err)
}
tree := NewMultiBPTree(p, uint64(p.PageSize()))
if err := tree.Reset(); err != nil {
t.Fatal(err)
}
Expand All @@ -46,8 +40,8 @@ func TestMultiBPTree(t *testing.T) {
t.Fatal("expected found")
}
mp := tree.MemoryPointer()
if mp.Length != 36 {
t.Fatalf("expected length 36, got %d", mp.Length)
if mp.Length != 24 {
t.Fatalf("expected length 24, got %d", mp.Length)
}
})

Expand All @@ -57,26 +51,23 @@ func TestMultiBPTree(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tree, err := NewMultiBPTree(p)
if err != nil {
t.Fatal(err)
}
tree := NewMultiBPTree(p, uint64(p.PageSize()))
if err := tree.Reset(); err != nil {
t.Fatal(err)
}
next1, err := tree.AddNext()
if err != nil {
t.Fatal(err)
}
if next1.MemoryPointer().Length != 36 {
t.Fatalf("expected length 36, got %d", next1)
if next1.MemoryPointer().Length != 24 {
t.Fatalf("expected length 24, got %d", next1)
}
next2, err := next1.AddNext()
if err != nil {
t.Fatal(err)
}
if next2.MemoryPointer().Length != 36 {
t.Fatalf("expected length 36, got %d", next2)
if next2.MemoryPointer().Length != 24 {
t.Fatalf("expected length 24, got %d", next2)
}

if next1.MemoryPointer().Offset == next2.MemoryPointer().Offset {
Expand All @@ -99,23 +90,61 @@ func TestMultiBPTree(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tree, err := NewMultiBPTree(p)
if err != nil {
t.Fatal(err)
}
tree := NewMultiBPTree(p, uint64(p.PageSize()))
if err := tree.Reset(); err != nil {
t.Fatal(err)
}
next1, err := tree.AddNext()
if err != nil {
t.Fatal(err)
}
if next1.MemoryPointer().Length != 36 {
t.Fatalf("expected length 36, got %d", next1)
if next1.MemoryPointer().Length != 24 {
t.Fatalf("expected length 24, got %d", next1)
}
_, err = tree.AddNext()
if err == nil {
t.Fatal("expected error")
}
})

t.Run("starts with empty metadata", func(t *testing.T) {
b := newSeekableBuffer()
p, err := NewPageFile(b)
if err != nil {
t.Fatal(err)
}
tree := NewMultiBPTree(p, uint64(p.PageSize()))
if err := tree.Reset(); err != nil {
t.Fatal(err)
}
metadata, err := tree.Metadata()
if err != nil {
t.Fatal(err)
}
if len(metadata) != 0 {
t.Fatalf("expected empty metadata, got %v", metadata)
}
})

t.Run("storing metadata works", func(t *testing.T) {
b := newSeekableBuffer()
p, err := NewPageFile(b)
if err != nil {
t.Fatal(err)
}
tree := NewMultiBPTree(p, uint64(p.PageSize()))
if err := tree.Reset(); err != nil {
t.Fatal(err)
}
if err := tree.SetMetadata([]byte("hello")); err != nil {
t.Fatal(err)
}
metadata, err := tree.Metadata()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(metadata, []byte("hello")) {
t.Fatalf("got %v want %v", metadata, []byte("hello"))
}
})
}
Loading

0 comments on commit ea9fbc7

Please sign in to comment.