Merge pull request #152 from lim-yoona/feat/AddHashMapIndex

Add HashMap in-memory index for FlyDB(#99)

engine/index/hashmap.go

@@ -0,0 +1,175 @@
+package index
+
+import (
+	"bytes"
+	"sort"
+	"sync"
+
+	"github.com/ByteStorage/FlyDB/engine/data"
+	"github.com/cornelk/hashmap"
+)
+
+/*
+hashmap index, encapsulates the hashmap library of cornelk
+*/
+
+// HashMap struct
+type HashMap struct {
+	hashmap *hashmap.Map[string, *data.LogRecordPst]
+	// To ensure Thread safety
+	// multi thread writes need to be locked
+	lock *sync.RWMutex
+}
+
+// NewHashMap create a hashmap index
+func NewHashMap() *HashMap {
+	return &HashMap{
+		hashmap: hashmap.New[string, *data.LogRecordPst](),
+		lock:    new(sync.RWMutex),
+	}
+}
+
+// Implement the methods of the index interface
+// Put stores the data location information of key into the index
+func (hm *HashMap) Put(key []byte, pst *data.LogRecordPst) bool {
+	hm.lock.Lock()
+	defer hm.lock.Unlock()
+
+	hm.hashmap.Set(string(key), pst)
+	return true
+}
+
+// Get gains the data location of the key in the index
+func (hm *HashMap) Get(key []byte) *data.LogRecordPst {
+	value, ok := hm.hashmap.Get(string(key))
+	if !ok {
+		return nil
+	}
+	return value
+}
+
+// Delete deletes data location of one key in index
+func (hm *HashMap) Delete(key []byte) bool {
+	hm.lock.Lock()
+	hm.lock.Unlock()
+
+	return hm.hashmap.Del(string(key))
+}
+
+// Size returns the size of the data in index
+func (hm *HashMap) Size() int {
+	return hm.hashmap.Len()
+}
+
+// Iterator returns a index Iterator
+func (hm *HashMap) Iterator(reverse bool) Iterator {
+	// if the HashMap is empty, returns a default iterator
+	if hm.hashmap == nil {
+		return NewDefaultHashMapIterator(reverse)
+	}
+	hm.lock.RLock()
+	defer hm.lock.RUnlock()
+	// if the HashMap is not empty, returns a iterator
+	return NewHashMapIterator(hm.hashmap, reverse)
+}
+
+// HashMapIterator struct
+type HashMapIterator struct {
+	currIndex int     // The subscript position of the current traversal
+	reverse   bool    // Whether it is reverse traversal
+	values    []*Item // Key + Location index information
+}
+
+// create a default HashMap Iterator for the empty HashMap
+func NewDefaultHashMapIterator(reverse bool) *HashMapIterator {
+	return &HashMapIterator{
+		currIndex: 0,
+		reverse:   reverse,
+		values:    nil,
+	}
+}
+
+// create a HashMapIterator
+func NewHashMapIterator(hm *hashmap.Map[string, *data.LogRecordPst], reverse bool) *HashMapIterator {
+	// Use values slice to store all data in values
+	values := make([]*Item, hm.Len())
+
+	// count the number of elements in the values slice
+	var count int = 0
+	// store all data into an slice values
+	// We use range() method in the hashmap implement to do this
+	// define an operator method
+	saveFunc := func(key string, value *data.LogRecordPst) bool {
+		count++
+		item := &Item{
+			key: []byte(key),
+			pst: value,
+		}
+		values = append(values, item)
+		return true
+	}
+	// call range() method
+	hm.Range(saveFunc)
+
+	// filter out nil values
+	values = values[count:]
+
+	// if reverse needed, reverse the slice
+	if reverse {
+		for i, j := 0, len(values)-1; i < j; i, j = i+1, j-1 {
+			values[i], values[j] = values[j], values[i]
+		}
+	}
+
+	return &HashMapIterator{
+		currIndex: 0,
+		reverse:   reverse,
+		values:    values,
+	}
+}
+
+// Rewind goes back to the begining of the Iterator,ie. the index of the first data
+func (hmIt *HashMapIterator) Rewind() {
+	hmIt.currIndex = 0
+}
+
+// Seek finds a >= or <= target key according to the incoming key,
+// and starts traversing from this target key
+func (hmIt *HashMapIterator) Seek(key []byte) {
+	if hmIt.reverse {
+		hmIt.currIndex = sort.Search(len(hmIt.values), func(i int) bool {
+			return bytes.Compare(hmIt.values[i].key, key) <= 0
+		})
+	} else {
+		hmIt.currIndex = sort.Search(len(hmIt.values), func(i int) bool {
+			return bytes.Compare(hmIt.values[i].key, key) >= 0
+		})
+	}
+}
+
+// Next jumps to the next key
+func (hmIt *HashMapIterator) Next() {
+	hmIt.currIndex += 1
+}
+
+// Valid refers to whether it is valid, that is,
+// whether all keys have been traversed,
+// used to exit the traverse ==> true->yes false-->no
+func (hmIt *HashMapIterator) Valid() bool {
+	return hmIt.currIndex < len(hmIt.values)
+}
+
+// Key returns the key data at the current traversal position
+func (hmIt *HashMapIterator) Key() []byte {
+	return hmIt.values[hmIt.currIndex].key
+}
+
+// Value returns the value data of the current traversal position
+func (hmIt *HashMapIterator) Value() *data.LogRecordPst {
+	return hmIt.values[hmIt.currIndex].pst
+}
+
+// Close closes the iterator and releases the corresponding resources
+func (hmIt *HashMapIterator) Close() {
+	hmIt.values = nil
+}

engine/index/hashmap_test.go

@@ -0,0 +1,96 @@
+package index
+
+import (
+	"testing"
+
+	"github.com/ByteStorage/FlyDB/engine/data"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHashMap_Put(t *testing.T) {
+	hm := NewHashMap()
+
+	res1 := hm.Put(nil, &data.LogRecordPst{Fid: 1, Offset: 100})
+	assert.True(t, res1)
+
+	res2 := hm.Put([]byte("a"), &data.LogRecordPst{Fid: 1, Offset: 200})
+	assert.True(t, res2)
+}
+
+func TestHashMap_Get(t *testing.T) {
+	hm := NewHashMap()
+
+	res1 := hm.Put(nil, &data.LogRecordPst{Fid: 1, Offset: 100})
+	assert.True(t, res1)
+
+	pst1 := hm.Get(nil)
+	assert.Equal(t, uint32(1), pst1.Fid)
+	assert.Equal(t, int64(100), pst1.Offset)
+
+	res2 := hm.Put([]byte("a"), &data.LogRecordPst{Fid: 1, Offset: 200})
+	assert.True(t, res2)
+	res3 := hm.Put([]byte("a"), &data.LogRecordPst{Fid: 1, Offset: 300})
+	assert.True(t, res3)
+
+	pst2 := hm.Get([]byte("a"))
+	assert.Equal(t, uint32(1), pst2.Fid)
+	assert.Equal(t, int64(300), pst2.Offset)
+}
+
+func TestHashMap_Delete(t *testing.T) {
+	hm := NewHashMap()
+
+	res1 := hm.Put(nil, &data.LogRecordPst{Fid: 1, Offset: 100})
+	assert.True(t, res1)
+	res2 := hm.Delete(nil)
+	assert.True(t, res2)
+
+	res3 := hm.Put([]byte("abc"), &data.LogRecordPst{Fid: 11, Offset: 22})
+	assert.True(t, res3)
+	res4 := hm.Delete([]byte("abc"))
+	assert.True(t, res4)
+}
+
+func TestHashMap_Iterator(t *testing.T) {
+	hm1 := NewHashMap()
+	// 1. HashMap is empty
+
+	iter1 := hm1.Iterator(false)
+	assert.Equal(t, false, iter1.Valid())
+
+	// 2. HashMap is not empty
+	hm1.Put([]byte("abc"), &data.LogRecordPst{Fid: 1, Offset: 12})
+
+	iter2 := hm1.Iterator(false)
+	assert.True(t, iter2.Valid())
+	assert.NotNil(t, iter2.Key())
+	assert.NotNil(t, iter2.Value())
+	iter2.Next()
+	assert.Equal(t, false, iter2.Valid())
+
+	// 3. when there are multiple pieces of data
+	hm1.Put([]byte("bcd"), &data.LogRecordPst{Fid: 2, Offset: 12})
+	hm1.Put([]byte("efg"), &data.LogRecordPst{Fid: 3, Offset: 12})
+	hm1.Put([]byte("def"), &data.LogRecordPst{Fid: 4, Offset: 12})
+	iter3 := hm1.Iterator(false)
+	for iter3.Rewind(); iter3.Valid(); iter3.Next() {
+		assert.NotNil(t, iter3.Key())
+	}
+
+	iter4 := hm1.Iterator(true)
+	for iter4.Rewind(); iter4.Valid(); iter4.Next() {
+		assert.NotNil(t, iter4.Key())
+	}
+
+	// 4. Seek test
+	iter5 := hm1.Iterator(false)
+	for iter5.Seek([]byte("b")); iter5.Valid(); iter5.Next() {
+		assert.NotNil(t, iter5.Key())
+	}
+
+	iter6 := hm1.Iterator(true)
+	for iter6.Seek([]byte("d")); iter6.Valid(); iter6.Next() {
+		assert.NotNil(t, iter6.Key())
+	}
+
+}

go.mod

@@ -3,6 +3,7 @@ module github.com/ByteStorage/FlyDB
 go 1.18
 
 require (
+	github.com/cornelk/hashmap v1.0.8
 	github.com/desertbit/grumble v1.1.3
 	github.com/fatih/color v1.13.0
 	github.com/google/btree v1.1.2

go.sum

@@ -23,6 +23,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWs
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
+github.com/cornelk/hashmap v1.0.8 h1:nv0AWgw02n+iDcawr5It4CjQIAcdMMKRrs10HOJYlrc=
+github.com/cornelk/hashmap v1.0.8/go.mod h1:RfZb7JO3RviW/rT6emczVuC/oxpdz4UsSB2LJSclR1k=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=