diff --git a/Makefile b/Makefile index 649b96e..3461fca 100644 --- a/Makefile +++ b/Makefile @@ -87,6 +87,9 @@ btree-test: oldv sql-test: oldv $(V) -stats $(BUILD_OPTIONS) test vsql/sql_test.v +orm-test: oldv + $(V) -stats $(BUILD_OPTIONS) test vsql/orm_test.v + # CLI Tests cli-test: bin/vsql diff --git a/docs/v-client-library-docs.rst b/docs/v-client-library-docs.rst index d4265f4..0f0f39d 100644 --- a/docs/v-client-library-docs.rst +++ b/docs/v-client-library-docs.rst @@ -13,15 +13,29 @@ Constants -fn open -------- +fn open_database +---------------- .. code-block:: v - pub fn open(path string) !&Connection + pub fn open_database(path string, options ConnectionOptions) !&Connection -open is the convenience function for open_database() with default options. +open_database will open an existing database file or create a new file if the path does not exist. + +If the file does exist, open_database will assume that the file is a valid database file (not corrupt). Otherwise unexpected behavior or even a crash may occur. + +The special file name ":memory:" can be used to create an entirely in-memory database. This will be faster but all data will be lost when the connection is closed. + +open_database can be used concurrently for reading and writing to the same file and provides the following default protections: + +- Fine: Multiple processes open_database() the same file. + +- Fine: Multiple goroutines sharing an open_database() on the same file. + +- Bad: Multiple goroutines open_database() the same file. + +See ConnectionOptions and default_connection_options(). fn catalog_name_from_path ------------------------- @@ -85,29 +99,15 @@ fn default_connection_options default_connection_options returns the sensible defaults used by open() and the correct base to provide your own option overrides. See ConnectionOptions. -fn open_database ----------------- +fn open_orm +----------- .. code-block:: v - pub fn open_database(path string, options ConnectionOptions) !&Connection - -open_database will open an existing database file or create a new file if the path does not exist. - -If the file does exist, open_database will assume that the file is a valid database file (not corrupt). Otherwise unexpected behavior or even a crash may occur. + pub fn open_orm(path string) !ORMConnection -The special file name ":memory:" can be used to create an entirely in-memory database. This will be faster but all data will be lost when the connection is closed. -open_database can be used concurrently for reading and writing to the same file and provides the following default protections: - -- Fine: Multiple processes open_database() the same file. - -- Fine: Multiple goroutines sharing an open_database() on the same file. - -- Bad: Multiple goroutines open_database() the same file. - -See ConnectionOptions and default_connection_options(). fn new_benchmark ---------------- @@ -217,6 +217,16 @@ new_numeric_value expects a value to be valid and the size and scale are determi +fn new_primitive_value +---------------------- + + +.. code-block:: v + + pub fn new_primitive_value(p orm.Primitive) !Value + +new_primitive_value returns the Value of a Primitive. Primitives are used by the ORM. + fn new_query_cache ------------------ @@ -287,25 +297,35 @@ fn new_timestamp_value new_timestamp_value creates a ``TIMESTAMP`` value. -fn new_varchar_value +fn new_unknown_value -------------------- .. code-block:: v - pub fn new_varchar_value(x string) Value + pub fn new_unknown_value() Value -new_varchar_value creates a ``CHARACTER VARYING`` value. +new_unknown_value returns an ``UNKNOWN`` value. This is the ``NULL`` representation of ``BOOLEAN``. -fn new_unknown_value +fn open +------- + + +.. code-block:: v + + pub fn open(path string) !&Connection + +open is the convenience function for open_database() with default options. + +fn new_varchar_value -------------------- .. code-block:: v - pub fn new_unknown_value() Value + pub fn new_varchar_value(x string) Value -new_unknown_value returns an ``UNKNOWN`` value. This is the ``NULL`` representation of ``BOOLEAN``. +new_varchar_value creates a ``CHARACTER VARYING`` value. type Row -------- @@ -341,6 +361,37 @@ enum Boolean Possible values for a BOOLEAN. +struct PageObject +----------------- + + +.. code-block:: v + + pub struct PageObject { + // The key is not required to be unique in the page. It becomes unique when + // combined with tid. However, no more than two version of the same key can + // exist in a page. See the caveats at the top of btree.v. + key []u8 + // The value contains the serialized data for the object. The first byte of + // key is used to both identify what type of object this is and also keep + // objects within the same collection also within the same range. + value []u8 + // When is_blob_ref is true, the value will be always be 5 bytes. See + // blob_info(). + is_blob_ref bool + mut: + // The tid is the transaction that created the object. + // + // TODO(elliotchance): It makes more sense to construct a new PageObject + // when changing the tid and xid. + tid int + // The xid is the transaciton that deleted the object, or zero if it has + // never been deleted. + xid int + } + +TODO(elliotchance): This does not need to be public. It was required for a bug at the time with V not being able to pass this to the shuffle function. At some point in the future remove the pub and see if it works. + struct ConnectionOptions ------------------------ @@ -684,36 +735,18 @@ struct Value A single value. It contains it's type information in ``typ``. -struct PageObject ------------------ +struct ORMConnection +-------------------- .. code-block:: v - pub struct PageObject { - // The key is not required to be unique in the page. It becomes unique when - // combined with tid. However, no more than two version of the same key can - // exist in a page. See the caveats at the top of btree.v. - key []u8 - // The value contains the serialized data for the object. The first byte of - // key is used to both identify what type of object this is and also keep - // objects within the same collection also within the same range. - value []u8 - // When is_blob_ref is true, the value will be always be 5 bytes. See - // blob_info(). - is_blob_ref bool + pub struct ORMConnection { mut: - // The tid is the transaction that created the object. - // - // TODO(elliotchance): It makes more sense to construct a new PageObject - // when changing the tid and xid. - tid int - // The xid is the transaciton that deleted the object, or zero if it has - // never been deleted. - xid int + c Connection } -TODO(elliotchance): This does not need to be public. It was required for a bug at the time with V not being able to pass this to the shuffle function. At some point in the future remove the pub and see if it works. + struct Identifier ----------------- diff --git a/examples/orm.v b/examples/orm.v new file mode 100644 index 0000000..9edc56a --- /dev/null +++ b/examples/orm.v @@ -0,0 +1,43 @@ +import os +import vsql + +fn main() { + os.rm('test.vsql') or {} + example() or { panic(err) } +} + +struct Product { + id int @[primary] + product_name string @[sql_type: 'varchar(100)'] + price f64 +} + +fn (p Product) str() string { + return '${p.product_name} ($${p.price})' +} + +fn example() ! { + mut db := vsql.open_orm('test.vsql')! + + sql db { + create table Product + }! + + products := [ + Product{1, 'Ice Cream', 5.99}, + Product{2, 'Ham Sandwhich', 3.47}, + Product{3, 'Bagel', 1.25}, + ] + for product in products { + sql db { + insert product into Product + }! + } + + println('Products over $2:') + for row in sql db { + select from Product where price > 2 + }! { + println(row) + } +} diff --git a/vsql/connection.v b/vsql/connection.v index 858e3aa..a7429fd 100644 --- a/vsql/connection.v +++ b/vsql/connection.v @@ -67,6 +67,10 @@ pub fn open(path string) !&Connection { return open_database(path, default_connection_options()) } +pub fn open_orm(path string) !ORMConnection { + return ORMConnection{open(path)!} +} + // open_database will open an existing database file or create a new file if the // path does not exist. // @@ -412,6 +416,23 @@ pub fn (mut conn CatalogConnection) schema_tables(schema string) ![]Table { return tables } +// schema_table returns the table for the provided schema. If the schema or +// table does not exist and empty list will be returned. +pub fn (mut conn CatalogConnection) schema_table(schema string, table string) !Table { + conn.open_read_connection()! + defer { + conn.release_read_connection() + } + + for _, t in conn.storage.tables { + if t.name.schema_name == schema && t.name.entity_name == table { + return t + } + } + + return sqlstate_42p01('table', table) // table does not exist +} + // resolve_identifier returns a new identifier that would represent the // canonical (fully qualified) form. fn (conn Connection) resolve_identifier(identifier Identifier) Identifier { diff --git a/vsql/earley.v b/vsql/earley.v index 47ed72b..44fd90f 100644 --- a/vsql/earley.v +++ b/vsql/earley.v @@ -237,7 +237,8 @@ fn parse(tokens []Token) !Stmt { mut columns := tokenize_earley_columns(tokens) mut grammar := get_grammar() - q0 := parse_earley(grammar[''], mut columns)! + q0 := parse_earley(grammar[''] or { panic('no entry rule') }, mut + columns)! trees := build_trees(q0) if trees.len == 0 { diff --git a/vsql/orm.v b/vsql/orm.v new file mode 100644 index 0000000..05cb00f --- /dev/null +++ b/vsql/orm.v @@ -0,0 +1,244 @@ +// orm.v implements the V language ORM: https://modules.vlang.io/orm.html + +module vsql + +import orm +import time + +pub struct ORMConnection { +mut: + c Connection +} + +fn (c ORMConnection) connection() Connection { + return c.c +} + +pub fn (c ORMConnection) @select(config orm.SelectConfig, data orm.QueryData, where orm.QueryData) ![][]orm.Primitive { + stmt := orm.orm_select_gen(config, '', true, ':p', 0, where) + + mut bound := map[string]Value{} + for i, _ in where.fields { + bound['p${i}'] = new_primitive_value(where.data[i])! + } + + mut prepared := unsafe { c.c.prepare(stmt)! } + result := prepared.query(bound)! + mut all_rows := [][]orm.Primitive{} + + for row in result { + mut primitive_row := []orm.Primitive{} + for column in result.columns { + primitive_row << row.get_primitive(column.name.str())! + } + + all_rows << primitive_row + } + + return all_rows +} + +pub fn (c ORMConnection) insert(table string, data orm.QueryData) ! { + values := data.data.map(fn (p orm.Primitive) string { + match p { + orm.InfixType { + // TODO(elliotchance): Not sure what this is? + return '${p}' + } + time.Time { + // TODO(elliotchance): This doesn't work. + return '${p}' + } + orm.Null { + return 'NULL' + } + bool { + if p { + return 'TRUE' + } + + return 'FALSE' + } + string { + // TODO(elliotchance): Does not escape correctly. + return '\'${p}\'' + } + f32 { + return '${p}' + } + f64 { + return '${p}' + } + i16 { + return '${p}' + } + i64 { + return '${p}' + } + i8 { + return '${p}' + } + int { + return '${p}' + } + u16 { + return '${p}' + } + u32 { + return '${p}' + } + u64 { + return '${p}' + } + u8 { + return '${p}' + } + } + }) + + unsafe { c.c.query('INSERT INTO ${table} (${data.fields.join(', ')}) VALUES (${values.join(', ')})')! } +} + +pub fn (c ORMConnection) update(table string, data orm.QueryData, where orm.QueryData) ! { + panic('update') +} + +pub fn (c ORMConnection) delete(table string, where orm.QueryData) ! { + panic('delete') +} + +pub fn (c ORMConnection) create(table string, fields []orm.TableField) ! { + mut sql_fields := []string{} + mut primary_key := '' + + for field in fields { + mut typ := orm_type_to_sql(field.name, field.typ, field.nullable)! + + // The double quotes and uppercase are required to make sure that reserved + // words are possible. + mut column_name := "\"${field.name.to_upper()}\"" + + for attr in field.attrs { + match attr.name { + 'sql' { + // "sql" is used to overload a bunch of different things. + if attr.arg == 'serial' { + primary_key = column_name + } else if orm.type_idx[attr.arg] != 0 { + typ = orm_type_to_sql(field.name, orm.type_idx[attr.arg], field.nullable)! + } else { + // Unlike above, we do not convert this to uppercase because we do not + // have the same V language naming limitations in the attribute. This + // means that in almost all cases you want the custom name to be in + // UPPERCASE if you want to use mixed cases in queries. + column_name = "\"${attr.arg}\"" + } + } + 'sql_type' { + typ = attr.arg + if field.nullable { '' } else { ' NOT NULL' } + } + 'primary' { + primary_key = column_name + } + 'unique' { + // Unique is not supported yet. It's better to throw an error so the + // data stays consistent. + return error('for ${field.name}: UNIQUE is not supported') + } + 'default' { + return error('for ${field.name}: DEFAULT is not supported') + } + else {} + } + } + + sql_fields << '${column_name} ${typ}' + } + + if primary_key != '' { + sql_fields << 'PRIMARY KEY (${primary_key})' + } + + // It's not possible to know if the table name has been generated from the + // struct name or extracted from @[table]. This creates a problem because we + // need to uppercase all generated names so they are safe to double-quote. + // However, this means that @[table] can never explicitly specify a case + // sensitive table name. The way around this is to require the table name to + // already be quoted if that was in the intention from @[table]. + table_name := if table[0] == `"` { table } else { requote_identifier(table.to_upper()) } + + create_table_sql := 'CREATE TABLE ${table_name} (${sql_fields.join(', ')})' + unsafe { + c.c.query(create_table_sql) or { return error('${err}: ${create_table_sql}') } + } +} + +fn orm_type_to_sql(field_name string, typ int, nullable bool) !string { + mut base_type := '' + + match typ { + orm.serial { + // This comes from @[sql: serial] which is supported, but it shouldn't + // come through this function as it's handled separately. Although if it + // does and we do need a type INTEGER should be a sensible default. + // + // NOT NULL is implied because this has to be part of the PRIMARY KEY. + return 'INTEGER NOT NULL' + } + orm.enum_ { + return error('for ${field_name}: ENUM is not supported') + } + orm.time_ { + // Let's choose the highest precision time possible (microseconds). + base_type = 'TIMESTAMP(6) WITH TIME ZONE' + } + orm.type_idx['i8'], orm.type_idx['u8'], orm.type_idx['i16'] { + base_type = 'SMALLINT' + } + orm.type_idx['u16'], orm.type_idx['int'] { + base_type = 'INTEGER' + } + orm.type_idx['i64'], orm.type_idx['u32'] { + base_type = 'BIGINT' + } + orm.type_idx['u64'] { + // u64 will not fit into a BIGINT so we have to use a larger exact type. + // It's worth noting that the database will allow a greater range than + // u64 (including negatives) but unless you're constructing SQL manually + // the u64 type will prevent you from doing this. + base_type = 'NUMERIC(20)' + } + orm.type_idx['f32'] { + base_type = 'REAL' + } + orm.type_idx['f64'] { + base_type = 'DOUBLE PRECISION' + } + orm.type_idx['bool'] { + base_type = 'BOOLEAN' + } + orm.type_idx['string'] { + base_type = 'VARCHAR(${orm.string_max_len})' + } + else { + // V's Type is an alias for int so we must include an else clause. There + // are also types that are not included in this switch that do not + // apply, or otherwise have no corresponding SQL type. + return error('unsupported type for ${field_name}: ${typ}') + } + } + + if !nullable { + return base_type + ' NOT NULL' + } + + return base_type +} + +pub fn (c ORMConnection) drop(table string) ! { + panic('drop') +} + +pub fn (c ORMConnection) last_id() int { + panic('last_id') +} diff --git a/vsql/orm_test.v b/vsql/orm_test.v new file mode 100644 index 0000000..9e4f4ef --- /dev/null +++ b/vsql/orm_test.v @@ -0,0 +1,190 @@ +module vsql + +import os +import time + +// ORMTable1 is for testing CREATE TABLE, it has many combinations of types and +// other attributes that are verified from the generated CREATE TABLE +// afterwards. +struct ORMTable1 { + // Each of the basic orm.Primative types without specifying any options. + a_bool bool // BOOLEAN + an_f32 f32 // REAL + an_f64 f64 // DOUBLE PRECISION + an_i16 i16 // SMALLINT + an_i64 i64 // BIGINT + an_i8 i8 // SMALLINT + an_int int // INTEGER + a_string string // CHARACTER VARYING(255) + a_time time.Time // TIMESTAMP(6) WITH TIME ZONE + an_u16 u16 // INTEGER + an_u32 u32 // BIGINT + an_u64 u64 // NUMERIC(20) + an_u8 u8 // SMALLINT + // Naming edge cases + where int // reserved word + actual_name int @[sql: 'secret_name'] + order int @[sql: 'ORDER'] + // Primary keys and other indexes. + a_primary_key int @[primary] // PRIMARY KEY (A_PRIMARY_KEY) + // Skipped fields + this_is_skipped int @[skip] + this_as_well int @[sql: '-'] + // Customize types + not_an_int int @[sql: string] + custom_sql_type int @[sql_type: 'NUMERIC(10)'] + // Nullable types + int_or_null ?int +} + +fn test_orm_create_success1() { + mut db := new_db() + sql db { + create table ORMTable1 + }! + + mut c := db.connection() + mut catalog := c.catalog() + mut table := catalog.schema_table('PUBLIC', 'ORMTABLE1')! + assert table.str() == 'CREATE TABLE "test"."PUBLIC".ORMTABLE1 ( + A_BOOL BOOLEAN NOT NULL, + AN_F32 REAL NOT NULL, + AN_F64 DOUBLE PRECISION NOT NULL, + AN_I16 SMALLINT NOT NULL, + AN_I64 BIGINT NOT NULL, + AN_I8 SMALLINT NOT NULL, + AN_INT INTEGER NOT NULL, + A_STRING CHARACTER VARYING(2048) NOT NULL, + A_TIME TIMESTAMP(6) WITH TIME ZONE NOT NULL, + AN_U16 INTEGER NOT NULL, + AN_U32 BIGINT NOT NULL, + AN_U64 NUMERIC(20) NOT NULL, + AN_U8 SMALLINT NOT NULL, + "WHERE" INTEGER NOT NULL, + "secret_name" INTEGER NOT NULL, + "ORDER" INTEGER NOT NULL, + A_PRIMARY_KEY INTEGER NOT NULL, + NOT_AN_INT CHARACTER VARYING(2048) NOT NULL, + CUSTOM_SQL_TYPE NUMERIC(10) NOT NULL, + INT_OR_NULL INTEGER, + PRIMARY KEY (A_PRIMARY_KEY) +);' +} + +// ORMTable2 tests some cases that are not possible on ORMTable1. Specifically: +// - A custom table name (that uses a reserved word to check quoting) +// - No primary key. +@[table: 'GROUP'] +struct ORMTable2 { + dummy int +} + +fn test_orm_create_success2() { + mut db := new_db() + sql db { + create table ORMTable2 + }! + + mut c := db.connection() + mut catalog := c.catalog() + assert catalog.schema_table('PUBLIC', 'GROUP')!.str() == 'CREATE TABLE "test"."PUBLIC"."GROUP" ( + DUMMY INTEGER NOT NULL +);' +} + +// ORMTableUnique makes sure we throw an error if @[unique] is used since it's +// not supported. +struct ORMTableUnique { + is_unique int @[unique] +} + +fn test_orm_create_unique_is_not_supported() { + mut db := new_db() + mut error := '' + sql db { + create table ORMTableUnique + } or { error = err.str() } + assert error == 'for is_unique: UNIQUE is not supported' +} + +// ORMTableSpecificName covers the edge case where already quoted table names +// need to remain intact. This is explained in more detail in create(). +// +// The extra escape is required for now, see bug +// https://github.com/vlang/v/issues/20313. +@[table: '\"specific name\"'] +struct ORMTableSpecificName { + dummy int +} + +fn test_orm_create_specific_name() { + mut db := new_db() + sql db { + create table ORMTableSpecificName + }! + + mut c := db.connection() + mut catalog := c.catalog() + assert catalog.schema_table('PUBLIC', 'specific name')!.str() == 'CREATE TABLE "test"."PUBLIC"."specific name" ( + DUMMY INTEGER NOT NULL +);' +} + +// ORMTableSerial lets the DB backend choose a column type for a auto-increment +// field. +struct ORMTableSerial { + dummy int @[sql: serial] +} + +fn test_orm_create_serial() { + mut db := new_db() + sql db { + create table ORMTableSerial + }! + + mut c := db.connection() + mut catalog := c.catalog() + assert catalog.schema_table('PUBLIC', 'ORMTABLESERIAL')!.str() == 'CREATE TABLE "test"."PUBLIC".ORMTABLESERIAL ( + DUMMY INTEGER NOT NULL, + PRIMARY KEY (DUMMY) +);' +} + +// ORMTableEnum is not supported. +struct ORMTableEnum { + an_enum Colors +} + +enum Colors { + red + green + blue +} + +fn test_orm_create_enum_is_not_supported() { + mut db := new_db() + mut error := '' + sql db { + create table ORMTableEnum + } or { error = err.str() } + assert error == 'for an_enum: ENUM is not supported' +} + +// ORMTableDefault is not supported. +struct ORMTableDefault { + has_default int @[default: '3'] +} + +fn test_orm_create_default_is_not_supported() { + mut db := new_db() + mut error := '' + sql db { + create table ORMTableDefault + } or { error = err.str() } + assert error == 'for has_default: DEFAULT is not supported' +} + +fn new_db() ORMConnection { + os.rm('test.vsql') or {} + return open_orm('test.vsql') or { panic(err) } +} diff --git a/vsql/row.v b/vsql/row.v index db37c30..61ea8cd 100644 --- a/vsql/row.v +++ b/vsql/row.v @@ -4,6 +4,7 @@ module vsql import time +import orm // Represents a single row which may contain one or more columns. struct Row { @@ -105,6 +106,13 @@ fn (r Row) for_storage() Row { return Row{r.id, r.tid, new_data} } +// primitives are used by the ORM. +pub fn (r Row) get_primitive(name string) !orm.Primitive { + value := r.get(name)! + + return value.primitive() +} + // new_empty_row is used internally to generate a row with zero values for all // the types in a Row. This is used for testing expressions without needing the // actual row. diff --git a/vsql/std_names_and_identifiers.v b/vsql/std_names_and_identifiers.v index 9ec7f3b..acf0b99 100644 --- a/vsql/std_names_and_identifiers.v +++ b/vsql/std_names_and_identifiers.v @@ -275,7 +275,7 @@ fn split_identifier_parts(s string) ![]string { } fn requote_identifier(s string) string { - if s.to_upper() == s { + if s.to_upper() == s && !is_key_word(s) { return s } diff --git a/vsql/table.v b/vsql/table.v index 6fda268..580f7d2 100644 --- a/vsql/table.v +++ b/vsql/table.v @@ -19,7 +19,7 @@ pub: // "foo" INT // BAR DOUBLE PRECISION NOT NULL pub fn (c Column) str() string { - mut f := '${c.name} ${c.typ}' + mut f := '${requote_identifier(c.name.sub_entity_name)} ${c.typ}' if c.not_null { f += ' NOT NULL' } @@ -136,6 +136,10 @@ pub fn (t Table) str() string { cols << ' ${col}' } + if t.primary_key.len > 0 { + cols << ' PRIMARY KEY (${t.primary_key.map(requote_identifier).join(', ')})' + } + return s + '\n' + cols.join(',\n') + '\n);' } diff --git a/vsql/value.v b/vsql/value.v index 0295d9f..ee6b9df 100644 --- a/vsql/value.v +++ b/vsql/value.v @@ -5,6 +5,7 @@ module vsql import regex +import orm // Possible values for a BOOLEAN. pub enum Boolean { @@ -186,6 +187,25 @@ pub fn new_numeric_value(x string) Value { } } +// new_primitive_value returns the Value of a Primitive. Primitives are used by +// the ORM. +pub fn new_primitive_value(p orm.Primitive) !Value { + return match p { + int { new_integer_value(p) } + f64 { new_double_precision_value(p) } + else { error('${p}') } + } +} + +fn bool_str(x f64) string { + return match x { + 0 { 'FALSE' } + 1 { 'TRUE' } + 2 { 'UNKNOWN' } + else { 'NULL' } + } +} + // new_decimal_value expects a value to be valid and the size and scale are // determined from the value as: // @@ -430,3 +450,14 @@ pub fn (v Value) numeric_value() Numeric { return v.v.numeric_value } } + +// primitives are used by the ORM. +pub fn (v Value) primitive() !orm.Primitive { + return match v.typ.typ { + // .is_boolean { orm.Primitive(v.f64_value() != 0) } + .is_integer { orm.Primitive(v.int_value()) } + .is_varchar { orm.Primitive(v.string_value()) } + .is_double_precision { orm.Primitive(v.f64_value()) } + else { error('${v.typ.typ}') } + } +}