Skip to content

Commit

Permalink
cache page data
Browse files Browse the repository at this point in the history
  • Loading branch information
@friendlymatthew
friendlymatthew committed Feb 12, 2024
1 parent 34c62af commit 2249aea
Show file tree
Hide file tree
Showing 6 changed files with 40 additions and 230 deletions.
6 changes: 3 additions & 3 deletions src/btree/bptree.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,14 +104,14 @@ export class BPTree {
const rootResponse = await this.root();

if (!rootResponse) {
return [{ offset: 0, length: 0 }, false];
return [{ offset: BigInt(0), length: 0 }, false];
}

let { rootNode, pointer } = rootResponse;

const path = await this.traverse(key, rootNode, pointer);
if (!path) {
return [{ offset: 0, length: 0 }, false];
return [{ offset: BigInt(0), length: 0 }, false];
}

const n = path[0].node;
Expand All @@ -122,7 +122,7 @@ export class BPTree {
return [n.pointers[i], true];
}

return [{ offset: 0, length: 0 }, false];
return [{ offset: BigInt(0), length: 0 }, false];
}
}

Expand Down
50 changes: 23 additions & 27 deletions src/btree/multi.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,19 @@ import { MemoryPointer } from "./node";
export class LinkedMetaPage {
private resolver: RangeResolver;
private offset: number;
private pageData: ArrayBuffer | null;

constructor(resolver: RangeResolver, offset: number) {
this.resolver = resolver;
this.offset = offset;
this.pageData = null;
}

async root(): Promise<MemoryPointer | null> {
// we seek by 12 bytes since offset is 8 bytes, length is 4 bytes
const { data } = await this.resolver({
start: this.offset,
end: this.offset + 12,
});

if (data.byteLength !== 12) {
return null;
}
const pageData = await this.getPage();

// we seek by 12 bytes since offset is 8 bytes, length is 4 bytes
const data = pageData.slice(this.offset, this.offset + 12);
const view = new DataView(data);

const pointerOffset = view.getBigUint64(0);
Expand All @@ -33,43 +29,43 @@ export class LinkedMetaPage {
}

async metadata(): Promise<ArrayBuffer> {
const { data: lengthData } = await this.resolver({
start: this.offset + 24,
end: this.offset + 28,
});
const pageData = await this.getPage();
const lengthData = pageData.slice(this.offset + 24, this.offset + 4096)

const lengthView = new DataView(lengthData);

// read the first four because that represnts length
const metadataLength = lengthView.getUint32(0);

// reard from 4 to 4 + length for metadata
const { data: metadata } = await this.resolver({
start: this.offset + 28,
end: this.offset + 28 + metadataLength,
});
const metadata = pageData.slice(this.offset + 28, this.offset + metadataLength);

return metadata;
}

async next(): Promise<LinkedMetaPage | null> {
private async getPage(): Promise<ArrayBuffer> {
if (this.pageData) {
return this.pageData
}

const { data } = await this.resolver({
start: this.offset + 12,
end: this.offset + 12 + 8,
start: this.offset,
end: this.offset + 4096 - 1,
});

if (data.byteLength !== 8) {
return null;
}
this.pageData = data;

return data;
}

async next(): Promise<LinkedMetaPage | null> {
const pageData = await this.getPage();
const data = pageData.slice(this.offset + 12, this.offset + 12 + 8);

const view = new DataView(data);

const nextOffset = view.getBigUint64(0);

// since 2^64 - 1
const maxUint64 = BigInt(2) ** BigInt(64) - BigInt(1);
if (nextOffset === maxUint64) {
// no next page
return null;
}

Expand Down
6 changes: 3 additions & 3 deletions src/btree/node.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ export class BPTreeNode {

node.keys.push({
value: Buffer.from(keyValue),
dataPointer: { offset: dpOffset, length: dpLength },
dataPointer: { offset: BigInt(dpOffset), length: dpLength },
});
} else {
let { data: keyValue } = await resolver({
Expand All @@ -77,7 +77,7 @@ export class BPTreeNode {

node.keys.push({
value: Buffer.from(keyValue),
dataPointer: { offset: currentOffset, length: l },
dataPointer: { offset: BigInt(currentOffset), length: l },
});

currentOffset += l;
Expand All @@ -104,7 +104,7 @@ export class BPTreeNode {
let pointerLength = lengthBuffer.readUint32BE(0);
currentOffset += 4;

node.pointers.push({ offset: pointerOffset, length: pointerLength });
node.pointers.push({ offset: BigInt(pointerOffset), length: pointerLength });
}

return { node, bytesRead: currentOffset };
Expand Down
19 changes: 0 additions & 19 deletions src/btree/pagefile.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,4 @@ export class PageFile {

return data;
}

async readFreePageIndex(): Promise<number[]> {
const { data } = await this.resolver({ start: 0, end: this.pageSize });
const view = new DataView(data);

const freePageIndexes = [];

for (let idx = 0; idx <= this.pageSize - 1; idx += 8) {
const offset = view.getBigUint64(idx);

if (offset === BigInt(0)) {
break; // the rest of the page is filled with zeros
}

freePageIndexes.push(Number(offset));
}

return freePageIndexes;
}
}
173 changes: 3 additions & 170 deletions src/index-file.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,10 @@ class IndexFileV1<T> implements VersionedIndexFile<T> {
if (this._tree) {
return this._tree;
}
// read the linked meta page

// we'll have to read the tree file: see multi.ts

this._tree = new LinkedMetaPage(this.resolver, 0);

return this._tree;
}

async metadata(): Promise<FileMeta | null> {
Expand All @@ -91,170 +91,3 @@ class IndexFileV1<T> implements VersionedIndexFile<T> {
};
}
}

/*
export interface VersionedIndexFile<T> {
indexFileHeader(): Promise<{
indexLength: number;
dataCount: number;
}>;
indexHeaders(): Promise<Header[]>;
indexRecord(
field: keyof T,
offset: number
): Promise<{
dataNumber: number;
fieldStartByteOffset: number;
fieldLength: number;
}>;
dataRecord(
offset: number
): Promise<{ startByteOffset: number; endByteOffset: number }>;
}
class IndexFileV1<T> implements VersionedIndexFile<T> {
private _indexFileHeader?: {
indexLength: number;
dataCount: number;
};
private _indexHeaders?: Header[];
private static INDEX_RECORD_SIZE = 18;
constructor(
private resolver: (start: number, end: number) => Promise<ArrayBuffer>
) {}
async indexFileHeader() {
if (this._indexFileHeader) {
return this._indexFileHeader;
}
const header = new DataView(await this.resolver(1, 16));
this._indexFileHeader = {
indexLength: Number(header.getBigUint64(0)),
dataCount: Number(header.getBigUint64(8)),
};
return this._indexFileHeader;
}
async indexHeaders() {
if (this._indexHeaders) {
return this._indexHeaders;
}
const indexFileHeader = await this.indexFileHeader();
const buffer = await this.resolver(17, indexFileHeader.indexLength + 16);
const data = new DataView(buffer);
const headers: Header[] = [];
let offset = 0;
while (offset < indexFileHeader.indexLength) {
const fieldNameLength = data.getUint32(offset);
offset += 4;
const fieldName = new TextDecoder("utf-8").decode(
buffer.slice(offset, offset + fieldNameLength)
);
offset += fieldNameLength;
const fieldType = data.getBigUint64(offset);
offset += 8;
const indexRecordCount = data.getBigUint64(offset);
offset += 8;
headers.push({
fieldName,
fieldType,
indexRecordCount,
});
}
if (offset !== indexFileHeader.indexLength) {
throw new Error(
`Inaccurate header read, offset = ${offset} but indexFileHeader.indexLength = ${indexFileHeader.indexLength}. This could indicate that the index file is corrupt.`
);
}
this._indexHeaders = headers;
return headers;
}
async indexRecord(field: keyof T, offset: number) {
if (offset < 0) {
throw new Error("offset out of range");
}
const headers = await this.indexHeaders();
const headerIndex = headers.findIndex(
(header) => header.fieldName === field
);
if (headerIndex === -1) {
throw new Error("field not found");
}
const header = headers[headerIndex];
if (offset >= Number(header.indexRecordCount)) {
throw new Error("offset out of range");
}
const indexFileHeader = await this.indexFileHeader();
const indexRecordsStart = 17 + indexFileHeader.indexLength;
const headerOffset = headers.slice(0, headerIndex).reduce((acc, header) => {
return (
acc + Number(header.indexRecordCount) * IndexFileV1.INDEX_RECORD_SIZE
);
}, 0);
const recordOffset =
indexRecordsStart + headerOffset + offset * IndexFileV1.INDEX_RECORD_SIZE;
const buffer = await this.resolver(
recordOffset,
recordOffset + IndexFileV1.INDEX_RECORD_SIZE
);
const data = new DataView(buffer);
const dataNumber = data.getBigUint64(0);
const fieldStartByteOffset = data.getBigUint64(8);
const fieldLength = decodeFloatingInt16(data.getUint16(16));
return {
dataNumber: Number(dataNumber),
fieldStartByteOffset: Number(fieldStartByteOffset),
fieldLength: fieldLength,
};
}
async dataRecord(offset: number) {
if (offset < 0) {
throw new Error("offset out of range");
}
const indexFileHeader = await this.indexFileHeader();
if (offset >= indexFileHeader.dataCount) {
throw new Error("offset out of range");
}
const headers = await this.indexHeaders();
const indexRecordsLength = headers.reduce((acc, header) => {
return (
acc + Number(header.indexRecordCount) * IndexFileV1.INDEX_RECORD_SIZE
);
}, 0);
const start = 17 + indexFileHeader.indexLength + indexRecordsLength;
// fetch the byte offsets. if offset is 0, we can just fetch the first 8 bytes.
if (offset === 0) {
const buffer = await this.resolver(
start + offset * 8,
start + offset * 8 + 8
);
const data = new DataView(buffer);
const endByteOffset = data.getBigUint64(0);
return {
startByteOffset: 0,
endByteOffset: Number(endByteOffset),
};
}
const buffer = await this.resolver(
start + (offset - 1) * 8,
start + offset * 8 + 8
);
const data = new DataView(buffer);
const startByteOffset = data.getBigUint64(0);
const endByteOffset = data.getBigUint64(8);
return {
startByteOffset: Number(startByteOffset),
endByteOffset: Number(endByteOffset),
};
}
}
*/
Loading

0 comments on commit 2249aea

Please sign in to comment.