Skip to content

Commit

Permalink
Add query validation to library (#77)
Browse files Browse the repository at this point in the history 
* add query validation

* refactor
  • Loading branch information
@friendlymatthew
friendlymatthew authored Jan 30, 2024
1 parent 0a5bc87 commit c439282
Show file tree
Hide file tree
Showing 7 changed files with 363 additions and 260 deletions.
131 changes: 17 additions & 114 deletions examples/client/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,13 @@
"green_tripdata_2023-01.csv.index",
Appendable.FormatType.Csv
).then(async (db) => {
let dbFields = new Set();
let fieldTypes = {};
let dbFields = [];
let queryHeaders = [];

// populate fields
db.fields().then((fields) => {
fields.map((field) => {
dbFields.add(field.fieldName);
fieldTypes[field.fieldName] = field.fieldType;
dbFields.push(field.fieldName);
});

document.getElementById("fields").innerHTML = JSON.stringify(
Expand All @@ -38,129 +36,30 @@
});

// then execute the query
document.getElementById("execute").onclick = () => {
document.getElementById("execute").onclick = async () => {
document.getElementById("results").innerHTML = "";
const queryJson = JSON.parse(editor.getValue());

const validationResult = validateQuery(
queryJson,
dbFields,
fieldTypes
);

if (validationResult !== "Valid Query") {
document.getElementById("results").innerHTML = validationResult;
return;
try {
const query = await db.query(queryJson);
let queryHeaders = queryJson.select ?? dbFields;
await bindQuery(query, queryHeaders);
} catch (error) {
console.log("error: ", error);
document.getElementById("results-header").innerHTML = "";
document.getElementById("results").innerHTML = error.message;
}

const query = db.query(queryJson);
let queryHeaders = queryJson.select ?? Array.from(dbFields);
bindQuery(query, queryHeaders);
};

document.getElementById("results").innerHTML = "";

const queryJson = JSON.parse(editor.getValue());
queryHeaders = queryJson.select ?? Array.from(dbFields);
queryHeaders = queryJson.select ?? dbFields;

const query = db.query(JSON.parse(editor.getValue()));
bindQuery(query, queryHeaders);
await bindQuery(query, queryHeaders);
});

function validateQuery(query, dbFields, fieldTypes) {
if (
!query.where ||
!Array.isArray(query.where) ||
query.where.length === 0
) {
return "Error: Missing 'where' clause.";
}

// validate the `where` clause
for (const whereNode of query.where) {
if (!["<", "<=", "==", ">=", ">"].includes(whereNode.operation)) {
return "Error: Invalid operation in 'where' clause.";
}
if (typeof whereNode.key !== "string") {
return "Error: 'key' in 'where' clause must be a string.";
}

if (!dbFields.has(whereNode.key)) {
return `Error: key: ${whereNode.key} in 'where' clause does not exist in dataset.`;
}

if (typeof whereNode.value === "undefined") {
return "Error: 'value' in 'where' clause is missing.";
}

const fieldType = fieldTypes[whereNode.key];

if (whereNode.value === null) {
if (
!Appendable.containsType(fieldType, Appendable.FieldType.Null)
) {
return `Error: 'key: ${whereNode.key} does not have type: null.`;
}
}
if (typeof whereNode.value === "boolean") {
if (
!Appendable.containsType(fieldType, Appendable.FieldType.Boolean)
) {
return `Error: 'key: ${whereNode.key} does not have type: boolean.`;
}
}
if (
typeof whereNode.value === "number" ||
typeof whereNode.value === "bigint"
) {
if (
!Appendable.containsType(fieldType, Appendable.FieldType.Number)
) {
return `Error: 'key: ${whereNode.key} does not have type: number.`;
}
}
if (typeof whereNode.value === "string") {
if (
!Appendable.containsType(fieldType, Appendable.FieldType.String)
) {
return `Error: 'key: ${whereNode.key} does not have type: string.`;
}
}
}

if (query.orderBy) {
// validate the `orderBy` clause
if (!Array.isArray(query.orderBy) || query.orderBy.length === 0) {
return "Error: Invalid 'orderby' clause.";
}

const orderBy = query.orderBy[0];

if (!["ASC", "DESC"].includes(orderBy.direction)) {
return "Error: Invalid direction in `orderBy`.";
}

if (orderBy.key !== query.where[0].key) {
return "Error: 'key' in `orderBy` must match `key` in `where` clause";
}
}

if (query.select) {
// validate the `selectFields` clause
if (!Array.isArray(query.select) || query.select.length === 0) {
return "Error: Invalid 'selectFields' clause.";
}

for (const field of query.select) {
if (!dbFields.has(field)) {
return `Error: 'key': ${field} in 'selectFields' clause does not exist in dataset.`;
}
}
}

return "Valid Query";
}

async function bindQuery(query, headers) {
const resultsHeaderElement = document.getElementById("results-header");
resultsHeaderElement.innerHTML = "";
Expand Down Expand Up @@ -237,6 +136,10 @@
max-height: calc(100vh - 50px);
overflow-y: auto;
}
#results {
overflow-y: auto;
max-height: calc(100vh - 670px);
}
#results-header {
width: max-content;
}
Expand Down
27 changes: 15 additions & 12 deletions src/database.ts → src/db/database.ts
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,24 @@
import { FormatType } from ".";
import { DataFile } from "./data-file";
import { IndexFile, VersionedIndexFile } from "./index-file";
import { FormatType } from "..";
import { DataFile } from "../data-file";
import { IndexFile, VersionedIndexFile } from "../index-file";
import { validateQuery } from "./query-validation";

type Schema = {
export type Schema = {
[key: string]: {};
};

type WhereNode<T extends Schema, K extends keyof T = keyof T> = {
export type WhereNode<T extends Schema, K extends keyof T = keyof T> = {
operation: "<" | "<=" | "==" | ">=" | ">";
key: keyof T;
value: T[K];
};

type OrderBy<T extends Schema> = {
export type OrderBy<T extends Schema> = {
key: keyof T;
direction: "ASC" | "DESC";
};

type SelectField<T extends Schema> = keyof T;
export type SelectField<T extends Schema> = keyof T;

export type Query<T extends Schema> = {
where?: WhereNode<T>[];
Expand All @@ -32,11 +33,6 @@ export enum FieldType {
Null = 1 << 5,
}

// given a fieldType and the desired type, this function performs a bitwise operation to test membership
export function containsType(fieldType: bigint, desiredType: FieldType) {
return (fieldType & BigInt(desiredType)) !== BigInt(0);
}

function parseIgnoringSuffix(
x: string,
format: FormatType,
Expand Down Expand Up @@ -151,6 +147,13 @@ export class Database<T extends Schema> {
// convert each of the where nodes into a range of field values.
const headers = await this.indexFile.indexHeaders();
const headerFields = headers.map((header) => header.fieldName);

try {
await validateQuery(query, headers);
} catch (error) {
throw new Error(`Query validation failed: ${(error as Error).message}`);
}

const fieldRanges = await Promise.all(
(query.where ?? []).map(async ({ key, value, operation }) => {
const header = headers.find((header) => header.fieldName === key);
Expand Down
Loading

0 comments on commit c439282

Please sign in to comment.