Merge branch 'main' into logger

examples/README.md

@@ -52,8 +52,8 @@ cp ../../dist/appendable.min.js.map ../client
 Then run the development server:
 
 ```sh
-npm run serve:example
+npm run example
 ```
 
 
-You should see the example built on http://192.168.1.157:8080
+You should see the example built on http://localhost:8080

examples/client/server.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"log"
+	"net/http"
+)
+
+func main() {
+	// Set the directory to serve
+	fs := http.FileServer(http.Dir("./"))
+
+	// Handle all requests by serving a file of the same name
+	http.Handle("/", fs)
+
+	// Define the port to listen on
+	port := "8080"
+	log.Printf("Listening on http://localhost:%s/", port)
+
+	// Start the server
+	err := http.ListenAndServe(":"+port, nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

package.json

@@ -6,7 +6,7 @@
 	"scripts": {
 		"build": "esbuild src/index.ts --bundle --minify --sourcemap --outfile=dist/appendable.min.js",
 		"build-index": "go run cmd/main.go -jsonl examples/workspace/green_tripdata_2023-01.jsonl",
-		"serve:example": "cd examples/client && npx http-server",
+		"example": "cd examples/client && go run server.go",
 		"test": "jest"
 	},
 	"repository": {
@@ -24,7 +24,6 @@
 	},
 	"devDependencies": {
 		"@types/jest": "^29.5.11",
-		"http-server": "^14.1.1",
 		"prettier": "^3.2.1",
 		"ts-jest": "^29.1.1",
 		"ts-node": "^10.9.2"

src/database.ts

@@ -35,7 +35,6 @@ export function containsType(fieldType: bigint, desiredType: FieldType) {
 }
 
 function parseIgnoringSuffix(x: string, format: FormatType) {
-	console.log("parseSuffix: ", x);
 	switch (format) {
 		case FormatType.Jsonl:
 			try {
@@ -54,9 +53,35 @@ function parseIgnoringSuffix(x: string, format: FormatType) {
 			return JSON.parse(x);
 
 		case FormatType.Csv:
+			try {
+				console.log("parsing no error", parseCsvLine(x));
+				return parseCsvLine(x);
+			} catch (error) {
+				console.log("registered as an error");
+				let lastCompleteLine = findLastCompleteCsvLine(x);
+				console.log(lastCompleteLine);
+				return parseCsvLine(lastCompleteLine);
+			}
 	}
 }
 
+export function parseCsvLine(line: string) {
+	console.log("parsing csv: ");
+	let fields: string[] = line.split(",");
+
+	fields.forEach((field) => {
+		if (field.length > 0) {
+			console.log("parsing: ", field);
+			return JSON.parse(field);
+		}
+	});
+}
+
+function findLastCompleteCsvLine(data: string) {
+	let lastNewlineIndex = data.lastIndexOf("\n");
+	return lastNewlineIndex >= 0 ? data.slice(0, lastNewlineIndex) : data;
+}
+
 function fieldRank(token: any) {
 	if (token === null) {
 		return 1;
@@ -182,6 +207,7 @@ export class Database<T extends Schema> {
 		);
 		// group the field ranges by the field name and merge them into single ranges.
 		const fieldRangeMap = new Map<keyof T, [number, number]>();
+
 		for (const [key, value] of fieldRanges) {
 			const existing = fieldRangeMap.get(key);
 			if (existing) {
@@ -207,6 +233,8 @@ export class Database<T extends Schema> {
 				fieldRangesSorted.unshift(...fieldRangesSorted.splice(index, 1));
 			}
 		}
+
+		console.log("Field ranges: ", fieldRanges);
 		// evaluate the field ranges in order.
 		for (const [key, [start, end]] of fieldRangesSorted) {
 			// check if the iteration order should be reversed.
@@ -219,6 +247,8 @@ export class Database<T extends Schema> {
 				const dataRecord = await this.indexFile.dataRecord(
 					indexRecord.dataNumber
 				);
+
+				console.log(`Data record: `, dataRecord);
 				const dataFieldValue = parseIgnoringSuffix(
 					await this.dataFile.get(
 						dataRecord.startByteOffset,

src/tests/database.test.ts

@@ -1,4 +1,4 @@
-import { Database, FieldType, Query, containsType } from "../database";
+import { Database, FieldType, Query, containsType, parseCsvLine } from "../database";
 import { DataFile } from "../data-file";
 import { IndexFile, VersionedIndexFile } from "../index-file";
 import { FormatType } from "..";
@@ -112,3 +112,20 @@ describe("test field type", () => {
 		});
 	});
 });
+
+
+describe("test parsing csv", () => {
+
+	it("check csv parse", async() => {
+		const testCases = [
+			{ data: "151,1", expected: 151},
+			{ data: ",95,5", expected: 95}
+		];
+
+		testCases.forEach(({ data, expected}) => {
+			let csv = parseCsvLine(data)	
+			console.log(csv)
+		})
+
+	})
+})