Skip to content

Commit

Permalink
Merge pull request #290 from skadefro/master
Browse files Browse the repository at this point in the history 
close 1.5.7
  • Loading branch information
@skadefro
skadefro authored Jan 3, 2024
2 parents 11f3170 + 9f3ae6a commit 1e3de6a
Show file tree
Hide file tree
Showing 30 changed files with 2,788 additions and 263 deletions.
3 changes: 2 additions & 1 deletion OpenFlow/src/Audit.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,14 +183,15 @@ export class Audit {
log.success = success;
log.months = months;
log.customerid = customerid;
log.domain = domain;
if(success) {
log.name = domain + " " + months + " months";
} else {
if(error != null && error != "") {
log.name = error;
} else {
log.name = domain + " failed";
}
}
}
log.username = username;
log.userid = userid;
Expand Down
8 changes: 6 additions & 2 deletions OpenFlow/src/Config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ export class dbConfig extends Base {
if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_redis_host", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval",
"NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url",
"domain", "enable_openapi", "enable_openapiauth", "ping_clients_interval", "tls_crt", "tls_key", "tls_ca",
"otel_metric_url", "otel_trace_url", "multi_tenant", "auto_hourly_housekeeping", "housekeeping_skip_calculate_size", "housekeeping_skip_update_user_size"].indexOf(key) > -1 ) {
"otel_metric_url", "otel_trace_url", "multi_tenant", "auto_hourly_housekeeping", "housekeeping_skip_calculate_size", "housekeeping_skip_update_user_size",
"stripe_api_secret", "stripe_api_key"].indexOf(key) > -1 ) {

if(os.hostname().toLowerCase() == "nixos") {
continue;
Expand Down Expand Up @@ -119,7 +120,8 @@ export class dbConfig extends Base {
if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_redis_host", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval",
"NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url",
"domain", "enable_openapi", "enable_openapiauth", "ping_clients_interval", "tls_crt", "tls_key", "tls_ca",
"otel_metric_url", "otel_trace_url", "multi_tenant", "auto_hourly_housekeeping", "housekeeping_skip_calculate_size", "housekeeping_skip_update_user_size" ].indexOf(key) > -1 ) {
"otel_metric_url", "otel_trace_url", "multi_tenant", "auto_hourly_housekeeping", "housekeeping_skip_calculate_size", "housekeeping_skip_update_user_size",
"stripe_api_secret", "stripe_api_key" ].indexOf(key) > -1 ) {
if(os.hostname().toLowerCase() == "nixos") {
continue;
}
Expand Down Expand Up @@ -222,6 +224,7 @@ export class Config {
enable_openai: false,
enable_openapi: true,
enable_openapiauth: true,
llmchat_queue: "",
log_with_colors: true,
cache_store_type: "memory",
cache_store_max: 1000,
Expand Down Expand Up @@ -490,6 +493,7 @@ export class Config {
public static enable_openai: boolean = Config.parseBoolean(Config.getEnv("enable_openai"));
public static enable_openapi: boolean = Config.parseBoolean(Config.getEnv("enable_openapi"));
public static enable_openapiauth: boolean = Config.parseBoolean(Config.getEnv("enable_openapiauth"));
public static llmchat_queue: string = Config.getEnv("llmchat_queue");
public static openai_token: string = Config.getEnv("openai_token");
public static version: string = Config.getversion();
public static log_with_colors: boolean = Config.parseBoolean(Config.getEnv("log_with_colors"));
Expand Down
2 changes: 1 addition & 1 deletion OpenFlow/src/DBHelper.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -579,7 +579,7 @@ export class DBHelper {
}
}
]
return Config.db.aggregate<User>(pipe, "users", Crypt.rootToken(), null, null, span);
return Config.db.aggregate<User>(pipe, "users", Crypt.rootToken(), null, null, false, span);
}
public DecorateWithRolesAllRolesWrap(span: Span) {
Logger.instanse.debug("Add all roles", span);
Expand Down
87 changes: 64 additions & 23 deletions OpenFlow/src/DatabaseConnection.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { MongoClient, ObjectId, Db, Binary, GridFSBucket, ChangeStream, MongoClientOptions, AggregateOptions, InsertOneOptions, InsertOneResult, UpdateOptions } from "mongodb";
import { MongoClient, ObjectId, Db, Binary, GridFSBucket, ChangeStream, MongoClientOptions, AggregateOptions, InsertOneOptions, InsertOneResult, UpdateOptions, FindOptions } from "mongodb";
import { Crypt } from "./Crypt";
import { Config, dbConfig } from "./Config";
import { TokenUser, Base, WellknownIds, Rights, NoderedUtil, mapFunc, finalizeFunc, reduceFunc, Ace, UpdateOneMessage, UpdateManyMessage, InsertOrUpdateOneMessage, Role, Rolemember, User, Customer, WatchEventMessage, Workitem, WorkitemQueue, QueryOptions, CountOptions, Resource, ResourceUsage } from "@openiap/openflow-api";
Expand Down Expand Up @@ -260,7 +260,6 @@ export class DatabaseConnection extends events.EventEmitter {
this.registerGlobalWatch(collections[c].name, span);
}
// }
this.ensureQueueMonitoring();
this.isConnected = true;
Logger.otel.endSpan(span);
this.emit("connected");
Expand Down Expand Up @@ -629,10 +628,12 @@ export class DatabaseConnection extends events.EventEmitter {
return false;
}
}
async ListCollections(jwt: string): Promise<any[]> {
async ListCollections(includesystem: boolean, jwt: string): Promise<any[]> {
// let result = await DatabaseConnection.toArray(this.db.listCollections());
let result = await Logger.DBHelper.GetCollections(null);
result = result.filter(x => x.name.indexOf("system.") === -1);
if(includesystem == false) {
result = result.filter(x => x.name.indexOf("system.") === -1);
}
result.sort((a, b) => a.name.localeCompare(b.name, undefined, { sensitivity: 'base' }))
await Crypt.verityToken(jwt);
return result;
Expand Down Expand Up @@ -1085,10 +1086,15 @@ export class DatabaseConnection extends events.EventEmitter {
for (let key of keys) {
if (key === "_id") {
const id: string = query._id;
const safeid = safeObjectID(id);
if (safeid !== null && safeid !== undefined) {
if(id.length == 12 || id.length == 24) {
const safeid = safeObjectID(id);
if (safeid !== null && safeid !== undefined) {
delete query._id;
query.$or = [{ _id: id }, { _id: safeObjectID(id) }];
}
} else {
delete query._id;
query.$or = [{ _id: id }, { _id: safeObjectID(id) }];
query.$or = [{ _id: id }];
}
}
}
Expand Down Expand Up @@ -1122,8 +1128,15 @@ export class DatabaseConnection extends events.EventEmitter {
span?.setAttribute("top", top);
span?.setAttribute("skip", skip);
let arr: T[] = [];
let findoptions: FindOptions = {};
// @ts-ignore
if(options.explain === true) {
console.log("explain quey");
// @ts-ignore
findoptions.explain = options.explain;
}
const ot_end = Logger.otel.startTimer();
let _pipe = this.db.collection(collectionname).find(_query);
let _pipe = this.db.collection(collectionname).find(_query, findoptions);
if (projection != null) {
_pipe = _pipe.project(projection);
}
Expand Down Expand Up @@ -1216,8 +1229,11 @@ export class DatabaseConnection extends events.EventEmitter {
const id: string = query._id;
const safeid = safeObjectID(id);
if (safeid !== null && safeid !== undefined) {
delete query._id;
query.$or = [{ _id: id }, { _id: safeObjectID(id) }];
const safeid = safeObjectID(id);
if (safeid !== null && safeid !== undefined) {
delete query._id;
query.$or = [{ _id: id }, { _id: safeObjectID(id) }];
}
}
}
}
Expand Down Expand Up @@ -1446,7 +1462,7 @@ export class DatabaseConnection extends events.EventEmitter {
* @param {string} jwt
* @returns Promise
*/
async aggregate<T extends Base>(aggregates: object[], collectionname: string, jwt: string, hint: Object | string, queryas:string, parent: Span): Promise<T[]> {
async aggregate<T extends Base>(aggregates: object[], collectionname: string, jwt: string, hint: Object | string, queryas:string, explain:boolean, parent: Span): Promise<T[]> {
const span: Span = Logger.otel.startSubSpan("db.Aggregate", parent);
await this.connect(span);
let json: any = aggregates;
Expand Down Expand Up @@ -1526,6 +1542,11 @@ export class DatabaseConnection extends events.EventEmitter {
aggregates.push({ "$limit": 500 });
}
const options: AggregateOptions = {};
if(explain === true) {
console.log("explain aggregate")
// @ts-ignore
options.explain = explain;
}
options.hint = myhint;
try {
const ot_end = Logger.otel.startTimer();
Expand Down Expand Up @@ -2907,10 +2928,14 @@ export class DatabaseConnection extends events.EventEmitter {

if (q.query === null || q.query === undefined) {
const id: string = q.item._id;
const safeid = safeObjectID(id);
q.query = { _id: id };
if (safeid != null) {
q.query = { $or: [{ _id: id }, { _id: safeid }] };
if(id.length == 12 || id.length == 24) {
const safeid = safeObjectID(id);
q.query = { _id: id };
if (safeid != null) {
q.query = { $or: [{ _id: id }, { _id: safeid }] };
}
} else {
q.query = { _id: id };
}
}
let _query: Object = {};
Expand Down Expand Up @@ -3139,8 +3164,13 @@ export class DatabaseConnection extends events.EventEmitter {
for (let key in q.query) {
if (key === "_id") {
const id: string = (q.query as any)._id;
delete (q.query as any)._id;
(q.query as any).$or = [{ _id: id }, { _id: safeObjectID(id) }];
if(id.length == 12 || id.length == 24) {
const safeid = safeObjectID(id);
if(safeid != null) {
delete (q.query as any)._id;
(q.query as any).$or = [{ _id: id }, { _id: safeObjectID(id) }];
}
}
}
}
let _query: Object = {};
Expand Down Expand Up @@ -3458,7 +3488,16 @@ export class DatabaseConnection extends events.EventEmitter {

if (collectionname === "files") { collectionname = "fs.files"; }
if (DatabaseConnection.usemetadata(collectionname)) {
_query = { $and: [{ _id: safeObjectID(id) }, this.getbasequery(user, [Rights.delete], collectionname)] };
if(id.length == 12 || id.length == 24) {
const safeid = safeObjectID(id);
if(safeid != null) {
_query = { $and: [{ _id: safeObjectID(id) }, this.getbasequery(user, [Rights.delete], collectionname)] };
} else {
_query = { $and: [{ _id: id }, this.getbasequery(user, [Rights.delete], collectionname)] };
}
} else {
_query = { $and: [{ _id: id }, this.getbasequery(user, [Rights.delete], collectionname)] };
}
const ot_end = Logger.otel.startTimer();
const cursor = this.db.collection(collectionname).find(_query);
const arr = await cursor.toArray();
Expand Down Expand Up @@ -3706,10 +3745,12 @@ export class DatabaseConnection extends events.EventEmitter {
for (let key of keys) {
if (key === "_id") {
const id: string = query._id;
const safeid = safeObjectID(id);
if (safeid !== null && safeid !== undefined) {
delete query._id;
query.$or = [{ _id: id }, { _id: safeObjectID(id) }];
if(id.length == 12 || id.length == 24) {
const safeid = safeObjectID(id);
if (safeid !== null && safeid !== undefined) {
delete query._id;
query.$or = [{ _id: id }, { _id: safeObjectID(id) }];
}
}
}
}
Expand Down Expand Up @@ -4473,7 +4514,7 @@ export class DatabaseConnection extends events.EventEmitter {
}
async createIndex(collectionname: string, name: string, keypath: any, options: any, parent: Span) {
const span: Span = Logger.otel.startSubSpan("db.createIndex", parent);
return new Promise((resolve, reject) => {
return new Promise<string>((resolve, reject) => {
try {
Logger.instanse.info("Adding index " + name + " to " + collectionname, span, { collection: collectionname });
if(typeof keypath === "string") keypath = JSON.parse(keypath);
Expand Down
1 change: 1 addition & 0 deletions OpenFlow/src/Logger.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,7 @@ export class Logger {
if (Config.log_cache) Logger.enabled["DBHelper"] = level.Verbose;
if (Config.log_amqp) Logger.enabled["amqpwrapper"] = level.Verbose;
if (Config.log_openapi) Logger.enabled["OpenAIProxy"] = level.Verbose;
if (Config.log_openapi) Logger.enabled["OpenAPIProxy"] = level.Verbose;

if (Config.log_login_provider) Logger.enabled["LoginProvider"] = level.Verbose;
if (Config.log_websocket) Logger.enabled["WebSocketServer"] = level.Verbose;
Expand Down
Loading

0 comments on commit 1e3de6a

Please sign in to comment.