indexfile tsreader (#97)

* init reader work

* cache page data

* fix tests, add traverse page

* comment out query, resolve types

src/bptree/bptree.ts → src/btree/bptree.ts

@@ -1,9 +1,6 @@
 import { BPTreeNode, MemoryPointer, compareBytes } from "./node";
 import { LengthIntegrityError, RangeResolver } from "../resolver";
-
-export interface MetaPage {
-	root(): Promise<MemoryPointer>;
-}
+import { LinkedMetaPage } from "./multi";
 
 type RootResponse = {
 	rootNode: BPTreeNode;
@@ -12,9 +9,9 @@ type RootResponse = {
 
 export class BPTree {
 	private tree: RangeResolver;
-	private meta: MetaPage;
+	private meta: LinkedMetaPage;
 
-	constructor(tree: RangeResolver, meta: MetaPage) {
+	constructor(tree: RangeResolver, meta: LinkedMetaPage) {
 		this.tree = tree;
 		this.meta = meta;
 	}
@@ -104,14 +101,14 @@ export class BPTree {
 		const rootResponse = await this.root();
 
 		if (!rootResponse) {
-			return [{ offset: 0, length: 0 }, false];
+			return [{ offset: BigInt(0), length: 0 }, false];
 		}
 
 		let { rootNode, pointer } = rootResponse;
 
 		const path = await this.traverse(key, rootNode, pointer);
 		if (!path) {
-			return [{ offset: 0, length: 0 }, false];
+			return [{ offset: BigInt(0), length: 0 }, false];
 		}
 
 		const n = path[0].node;
@@ -122,7 +119,7 @@ export class BPTree {
 			return [n.pointers[i], true];
 		}
 
-		return [{ offset: 0, length: 0 }, false];
+		return [{ offset: BigInt(0), length: 0 }, false];
 	}
 }
 

src/btree/multi.ts

@@ -0,0 +1,77 @@
+import { RangeResolver } from "../resolver";
+import { MemoryPointer } from "./node";
+
+
+const PAGE_SIZE_BYTES = 4096;
+
+export class LinkedMetaPage {
+	private resolver: RangeResolver;
+	private offset: number;
+	private pageData: ArrayBuffer | null;
+
+	constructor(resolver: RangeResolver, offset: number) {
+		this.resolver = resolver;
+		this.offset = offset;
+		this.pageData = null;
+	}
+
+	async root(): Promise<MemoryPointer | null> {
+		const pageData = await this.getPage();
+
+		// we seek by 12 bytes since offset is 8 bytes, length is 4 bytes
+		const data = pageData.slice(this.offset, this.offset + 12);
+		const view = new DataView(data);
+
+		const pointerOffset = view.getBigUint64(0);
+		const lengthOffset = view.getUint32(8);
+
+		return {
+			offset: pointerOffset,
+			length: lengthOffset,
+		};
+	}
+
+	async metadata(): Promise<ArrayBuffer> {
+		const pageData = await this.getPage();	
+		const lengthData = pageData.slice(this.offset + 24, this.offset + PAGE_SIZE_BYTES)	
+
+		const lengthView = new DataView(lengthData);
+
+		// read the first four because that represnts length
+		const metadataLength = lengthView.getUint32(0);
+		const metadata = pageData.slice(this.offset + 28, this.offset + metadataLength);
+
+		return metadata;
+	}
+
+	private async getPage(): Promise<ArrayBuffer> {
+		if (this.pageData) {
+			return this.pageData
+		}	
+
+		const { data } = await this.resolver({
+			start: this.offset,
+			end: this.offset + PAGE_SIZE_BYTES - 1,
+		});
+
+		this.pageData = data;
+
+		return data;
+	}
+
+	async next(): Promise<LinkedMetaPage | null> {
+		const pageData = await this.getPage();
+		const data = pageData.slice(this.offset + 12, this.offset + 12 + 8);
+
+		const view = new DataView(data);
+
+		const nextOffset = view.getBigUint64(0);
+
+		const maxUint64 = BigInt(2) ** BigInt(64) - BigInt(1);
+		if (nextOffset === maxUint64) {
+			return null;
+		}
+
+		return new LinkedMetaPage(this.resolver, Number(nextOffset));
+	}
+}

src/bptree/node.ts → src/btree/node.ts

@@ -1,7 +1,7 @@
 import { RangeResolver } from "../resolver";
 
 export type ReferencedValue = { dataPointer: MemoryPointer; value: Buffer };
-export type MemoryPointer = { offset: number; length: number };
+export type MemoryPointer = { offset: BigInt; length: number };
 
 export class BPTreeNode {
 	public keys: ReferencedValue[];
@@ -23,8 +23,8 @@ export class BPTreeNode {
 		try {
 			const node = new BPTreeNode([], []);
 			let { data: sizeData } = await resolver({
-				start: mp.offset,
-				end: mp.offset + mp.length,
+				start: Number(mp.offset),
+				end: Number(mp.offset) + mp.length,
 			});
 
 			let sizeBuffer = Buffer.from(sizeData);
@@ -67,7 +67,7 @@ export class BPTreeNode {
 
 					node.keys.push({
 						value: Buffer.from(keyValue),
-						dataPointer: { offset: dpOffset, length: dpLength },
+						dataPointer: { offset: BigInt(dpOffset), length: dpLength },
 					});
 				} else {
 					let { data: keyValue } = await resolver({
@@ -77,7 +77,7 @@ export class BPTreeNode {
 
 					node.keys.push({
 						value: Buffer.from(keyValue),
-						dataPointer: { offset: currentOffset, length: l },
+						dataPointer: { offset: BigInt(currentOffset), length: l },
 					});
 
 					currentOffset += l;
@@ -104,7 +104,7 @@ export class BPTreeNode {
 				let pointerLength = lengthBuffer.readUint32BE(0);
 				currentOffset += 4;
 
-				node.pointers.push({ offset: pointerOffset, length: pointerLength });
+				node.pointers.push({ offset: BigInt(pointerOffset), length: pointerLength });
 			}
 
 			return { node, bytesRead: currentOffset };

src/db/database.ts

@@ -1,6 +1,6 @@
 import { FormatType } from "..";
 import { DataFile } from "../data-file";
-import { IndexFile, VersionedIndexFile } from "../index-file";
+import { IndexFile, VersionedIndexFile } from "../index-file/index-file";
 import { QueryBuilder } from "./query-builder";
 import { validateQuery } from "./query-validation";
 
@@ -141,6 +141,7 @@ export class Database<T extends Schema> {
 		return await this.indexFile.indexHeaders();
 	}
 
+	/*
 	async *query(query: Query<T>) {
 		// verify that the query does not require a composite index
 		if (new Set((query.where ?? []).map((where) => where.key)).size > 1) {
@@ -296,6 +297,7 @@ export class Database<T extends Schema> {
 			}
 		}
 	}
+	*/
 
 	where(
 		key: keyof T,

src/db/query-builder.ts

@@ -40,7 +40,7 @@ export class QueryBuilder<T extends Schema> {
 	 * Executes the constructed query
 	 */
 	get() {
-		return this.database.query(this.queryObject);
+		//return this.database.query(this.queryObject);
 	}
 
 	/**

src/db/query-validation.ts

@@ -1,4 +1,4 @@
-import { Header } from "../index-file";
+import { IndexMeta } from "../index-file/index-meta";
 import {
 	FieldType,
 	OrderBy,
@@ -24,12 +24,12 @@ function containsType(compositeType: bigint, singleType: FieldType): boolean {
  * validateWhere validates the 'where' clause of the query.
  *
  * @param {WhereNode<T>[] | undefined} where - The 'where' clause to validate.
- * @param {Header[]} headers - List of headers to check field existence and type compatibility.
+ * @param {IndexMeta[]} headers - List of headers to check field existence and type compatibility.
  * @throws {Error} - Throws an error if the 'where' clause is missing, invalid, or refers to non-existent fields.
  */
 function validateWhere<T extends Schema>(
 	where: WhereNode<T>[] | undefined,
-	headers: Header[]
+	headers: IndexMeta[]
 ): void {
 	if (!where || !Array.isArray(where) || where.length === 0) {
 		throw new Error("Missing 'where' clause.");
@@ -138,12 +138,12 @@ function validateOrderBy<T extends Schema>(
  * validateSelect validates the 'select' fields of a query.
  *
  * @param {SelectField<T>[] | undefined} select - The 'select' clause to validate.
- * @param {Header[]} headers - List of headers to check for field existence.
+ * @param {IndexMeta[]} headers - List of headers to check for field existence.
  * @throws {Error} Throws an error if any field in the 'select' clause doesn't exist in headers.
  */
 function validateSelect<T extends Schema>(
 	select: SelectField<T>[] | undefined,
-	headers: Header[]
+	headers: IndexMeta[]
 ): void {
 	if (select) {
 		console.log("validating select: ", select);
@@ -169,12 +169,12 @@ function validateSelect<T extends Schema>(
  * If any part of the query is invalid (e.g., a field doesn't exist), it throws an error.
  *
  * @param {Query<T>} query - The query object to validate.
- * @param {Header[]} headers - The headers against which to validate the query fields.
+ * @param {IndexMeta[]} headers - The headers against which to validate the query fields.
  * @throws {Error} Throws an error if query is invalid.
  */
 export async function validateQuery<T extends Schema>(
 	query: Query<T>,
-	headers: Header[]
+	headers: IndexMeta[]
 ): Promise<void> {
 	validateWhere(query.where, headers);
 	validateOrderBy(query.orderBy, query.where![0].key as string);