diff --git a/OpenFlow/src/Audit.ts b/OpenFlow/src/Audit.ts index 0aee7243..dea10863 100644 --- a/OpenFlow/src/Audit.ts +++ b/OpenFlow/src/Audit.ts @@ -183,6 +183,7 @@ export class Audit { log.success = success; log.months = months; log.customerid = customerid; + log.domain = domain; if(success) { log.name = domain + " " + months + " months"; } else { @@ -190,7 +191,7 @@ export class Audit { log.name = error; } else { log.name = domain + " failed"; - } + } } log.username = username; log.userid = userid; diff --git a/OpenFlow/src/Config.ts b/OpenFlow/src/Config.ts index bc3c21dd..a8647071 100644 --- a/OpenFlow/src/Config.ts +++ b/OpenFlow/src/Config.ts @@ -68,7 +68,8 @@ export class dbConfig extends Base { if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_redis_host", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "domain", "enable_openapi", "enable_openapiauth", "ping_clients_interval", "tls_crt", "tls_key", "tls_ca", - "otel_metric_url", "otel_trace_url", "multi_tenant", "auto_hourly_housekeeping", "housekeeping_skip_calculate_size", "housekeeping_skip_update_user_size"].indexOf(key) > -1 ) { + "otel_metric_url", "otel_trace_url", "multi_tenant", "auto_hourly_housekeeping", "housekeeping_skip_calculate_size", "housekeeping_skip_update_user_size", + "stripe_api_secret", "stripe_api_key"].indexOf(key) > -1 ) { if(os.hostname().toLowerCase() == "nixos") { continue; @@ -119,7 +120,8 @@ export class dbConfig extends Base { if(["license_key", "otel_trace_url", "cache_store_type", "cache_store_redis_host", "cache_store_max", "grafana_url", "workitem_queue_monitoring_interval", "NODE_ENV", "validate_emails", "amqp_url", "port", "saml_issuer", "saml_federation_metadata", "api_ws_url", "domain", "enable_openapi", "enable_openapiauth", "ping_clients_interval", "tls_crt", "tls_key", "tls_ca", - "otel_metric_url", "otel_trace_url", "multi_tenant", "auto_hourly_housekeeping", "housekeeping_skip_calculate_size", "housekeeping_skip_update_user_size" ].indexOf(key) > -1 ) { + "otel_metric_url", "otel_trace_url", "multi_tenant", "auto_hourly_housekeeping", "housekeeping_skip_calculate_size", "housekeeping_skip_update_user_size", + "stripe_api_secret", "stripe_api_key" ].indexOf(key) > -1 ) { if(os.hostname().toLowerCase() == "nixos") { continue; } @@ -222,6 +224,7 @@ export class Config { enable_openai: false, enable_openapi: true, enable_openapiauth: true, + llmchat_queue: "", log_with_colors: true, cache_store_type: "memory", cache_store_max: 1000, @@ -490,6 +493,7 @@ export class Config { public static enable_openai: boolean = Config.parseBoolean(Config.getEnv("enable_openai")); public static enable_openapi: boolean = Config.parseBoolean(Config.getEnv("enable_openapi")); public static enable_openapiauth: boolean = Config.parseBoolean(Config.getEnv("enable_openapiauth")); + public static llmchat_queue: string = Config.getEnv("llmchat_queue"); public static openai_token: string = Config.getEnv("openai_token"); public static version: string = Config.getversion(); public static log_with_colors: boolean = Config.parseBoolean(Config.getEnv("log_with_colors")); diff --git a/OpenFlow/src/DBHelper.ts b/OpenFlow/src/DBHelper.ts index 7c411d1d..55e2c6ba 100644 --- a/OpenFlow/src/DBHelper.ts +++ b/OpenFlow/src/DBHelper.ts @@ -579,7 +579,7 @@ export class DBHelper { } } ] - return Config.db.aggregate(pipe, "users", Crypt.rootToken(), null, null, span); + return Config.db.aggregate(pipe, "users", Crypt.rootToken(), null, null, false, span); } public DecorateWithRolesAllRolesWrap(span: Span) { Logger.instanse.debug("Add all roles", span); diff --git a/OpenFlow/src/DatabaseConnection.ts b/OpenFlow/src/DatabaseConnection.ts index 46375480..bd16ac9f 100644 --- a/OpenFlow/src/DatabaseConnection.ts +++ b/OpenFlow/src/DatabaseConnection.ts @@ -1,4 +1,4 @@ -import { MongoClient, ObjectId, Db, Binary, GridFSBucket, ChangeStream, MongoClientOptions, AggregateOptions, InsertOneOptions, InsertOneResult, UpdateOptions } from "mongodb"; +import { MongoClient, ObjectId, Db, Binary, GridFSBucket, ChangeStream, MongoClientOptions, AggregateOptions, InsertOneOptions, InsertOneResult, UpdateOptions, FindOptions } from "mongodb"; import { Crypt } from "./Crypt"; import { Config, dbConfig } from "./Config"; import { TokenUser, Base, WellknownIds, Rights, NoderedUtil, mapFunc, finalizeFunc, reduceFunc, Ace, UpdateOneMessage, UpdateManyMessage, InsertOrUpdateOneMessage, Role, Rolemember, User, Customer, WatchEventMessage, Workitem, WorkitemQueue, QueryOptions, CountOptions, Resource, ResourceUsage } from "@openiap/openflow-api"; @@ -260,7 +260,6 @@ export class DatabaseConnection extends events.EventEmitter { this.registerGlobalWatch(collections[c].name, span); } // } - this.ensureQueueMonitoring(); this.isConnected = true; Logger.otel.endSpan(span); this.emit("connected"); @@ -629,10 +628,12 @@ export class DatabaseConnection extends events.EventEmitter { return false; } } - async ListCollections(jwt: string): Promise { + async ListCollections(includesystem: boolean, jwt: string): Promise { // let result = await DatabaseConnection.toArray(this.db.listCollections()); let result = await Logger.DBHelper.GetCollections(null); - result = result.filter(x => x.name.indexOf("system.") === -1); + if(includesystem == false) { + result = result.filter(x => x.name.indexOf("system.") === -1); + } result.sort((a, b) => a.name.localeCompare(b.name, undefined, { sensitivity: 'base' })) await Crypt.verityToken(jwt); return result; @@ -1085,10 +1086,15 @@ export class DatabaseConnection extends events.EventEmitter { for (let key of keys) { if (key === "_id") { const id: string = query._id; - const safeid = safeObjectID(id); - if (safeid !== null && safeid !== undefined) { + if(id.length == 12 || id.length == 24) { + const safeid = safeObjectID(id); + if (safeid !== null && safeid !== undefined) { + delete query._id; + query.$or = [{ _id: id }, { _id: safeObjectID(id) }]; + } + } else { delete query._id; - query.$or = [{ _id: id }, { _id: safeObjectID(id) }]; + query.$or = [{ _id: id }]; } } } @@ -1122,8 +1128,15 @@ export class DatabaseConnection extends events.EventEmitter { span?.setAttribute("top", top); span?.setAttribute("skip", skip); let arr: T[] = []; + let findoptions: FindOptions = {}; + // @ts-ignore + if(options.explain === true) { + console.log("explain quey"); + // @ts-ignore + findoptions.explain = options.explain; + } const ot_end = Logger.otel.startTimer(); - let _pipe = this.db.collection(collectionname).find(_query); + let _pipe = this.db.collection(collectionname).find(_query, findoptions); if (projection != null) { _pipe = _pipe.project(projection); } @@ -1216,8 +1229,11 @@ export class DatabaseConnection extends events.EventEmitter { const id: string = query._id; const safeid = safeObjectID(id); if (safeid !== null && safeid !== undefined) { - delete query._id; - query.$or = [{ _id: id }, { _id: safeObjectID(id) }]; + const safeid = safeObjectID(id); + if (safeid !== null && safeid !== undefined) { + delete query._id; + query.$or = [{ _id: id }, { _id: safeObjectID(id) }]; + } } } } @@ -1446,7 +1462,7 @@ export class DatabaseConnection extends events.EventEmitter { * @param {string} jwt * @returns Promise */ - async aggregate(aggregates: object[], collectionname: string, jwt: string, hint: Object | string, queryas:string, parent: Span): Promise { + async aggregate(aggregates: object[], collectionname: string, jwt: string, hint: Object | string, queryas:string, explain:boolean, parent: Span): Promise { const span: Span = Logger.otel.startSubSpan("db.Aggregate", parent); await this.connect(span); let json: any = aggregates; @@ -1526,6 +1542,11 @@ export class DatabaseConnection extends events.EventEmitter { aggregates.push({ "$limit": 500 }); } const options: AggregateOptions = {}; + if(explain === true) { + console.log("explain aggregate") + // @ts-ignore + options.explain = explain; + } options.hint = myhint; try { const ot_end = Logger.otel.startTimer(); @@ -2907,10 +2928,14 @@ export class DatabaseConnection extends events.EventEmitter { if (q.query === null || q.query === undefined) { const id: string = q.item._id; - const safeid = safeObjectID(id); - q.query = { _id: id }; - if (safeid != null) { - q.query = { $or: [{ _id: id }, { _id: safeid }] }; + if(id.length == 12 || id.length == 24) { + const safeid = safeObjectID(id); + q.query = { _id: id }; + if (safeid != null) { + q.query = { $or: [{ _id: id }, { _id: safeid }] }; + } + } else { + q.query = { _id: id }; } } let _query: Object = {}; @@ -3139,8 +3164,13 @@ export class DatabaseConnection extends events.EventEmitter { for (let key in q.query) { if (key === "_id") { const id: string = (q.query as any)._id; - delete (q.query as any)._id; - (q.query as any).$or = [{ _id: id }, { _id: safeObjectID(id) }]; + if(id.length == 12 || id.length == 24) { + const safeid = safeObjectID(id); + if(safeid != null) { + delete (q.query as any)._id; + (q.query as any).$or = [{ _id: id }, { _id: safeObjectID(id) }]; + } + } } } let _query: Object = {}; @@ -3458,7 +3488,16 @@ export class DatabaseConnection extends events.EventEmitter { if (collectionname === "files") { collectionname = "fs.files"; } if (DatabaseConnection.usemetadata(collectionname)) { - _query = { $and: [{ _id: safeObjectID(id) }, this.getbasequery(user, [Rights.delete], collectionname)] }; + if(id.length == 12 || id.length == 24) { + const safeid = safeObjectID(id); + if(safeid != null) { + _query = { $and: [{ _id: safeObjectID(id) }, this.getbasequery(user, [Rights.delete], collectionname)] }; + } else { + _query = { $and: [{ _id: id }, this.getbasequery(user, [Rights.delete], collectionname)] }; + } + } else { + _query = { $and: [{ _id: id }, this.getbasequery(user, [Rights.delete], collectionname)] }; + } const ot_end = Logger.otel.startTimer(); const cursor = this.db.collection(collectionname).find(_query); const arr = await cursor.toArray(); @@ -3706,10 +3745,12 @@ export class DatabaseConnection extends events.EventEmitter { for (let key of keys) { if (key === "_id") { const id: string = query._id; - const safeid = safeObjectID(id); - if (safeid !== null && safeid !== undefined) { - delete query._id; - query.$or = [{ _id: id }, { _id: safeObjectID(id) }]; + if(id.length == 12 || id.length == 24) { + const safeid = safeObjectID(id); + if (safeid !== null && safeid !== undefined) { + delete query._id; + query.$or = [{ _id: id }, { _id: safeObjectID(id) }]; + } } } } @@ -4473,7 +4514,7 @@ export class DatabaseConnection extends events.EventEmitter { } async createIndex(collectionname: string, name: string, keypath: any, options: any, parent: Span) { const span: Span = Logger.otel.startSubSpan("db.createIndex", parent); - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { try { Logger.instanse.info("Adding index " + name + " to " + collectionname, span, { collection: collectionname }); if(typeof keypath === "string") keypath = JSON.parse(keypath); diff --git a/OpenFlow/src/Logger.ts b/OpenFlow/src/Logger.ts index d74ccfc6..2fad4e75 100644 --- a/OpenFlow/src/Logger.ts +++ b/OpenFlow/src/Logger.ts @@ -329,6 +329,7 @@ export class Logger { if (Config.log_cache) Logger.enabled["DBHelper"] = level.Verbose; if (Config.log_amqp) Logger.enabled["amqpwrapper"] = level.Verbose; if (Config.log_openapi) Logger.enabled["OpenAIProxy"] = level.Verbose; + if (Config.log_openapi) Logger.enabled["OpenAPIProxy"] = level.Verbose; if (Config.log_login_provider) Logger.enabled["LoginProvider"] = level.Verbose; if (Config.log_websocket) Logger.enabled["WebSocketServer"] = level.Verbose; diff --git a/OpenFlow/src/LoginProvider.ts b/OpenFlow/src/LoginProvider.ts index 8934dec1..99caa6a6 100644 --- a/OpenFlow/src/LoginProvider.ts +++ b/OpenFlow/src/LoginProvider.ts @@ -94,12 +94,12 @@ export class LoginProvider { const options = { publicKey: Buffer.from(Config.signing_crt, "base64").toString("ascii") } - Logger.instanse.verbose("saml.validate", span); + Logger.instanse.verbose("saml.validate", span, {cls: "LoginProvider", func: "validateToken"}); saml.validate(rawAssertion, options, async (err, profile) => { const span: Span = Logger.otel.startSpan("saml.validate", null, null); try { if (err) { - Logger.instanse.error(err, span); + Logger.instanse.error(err, span, {cls: "LoginProvider", func: "validateToken"}); return reject(err); } const claims = profile.claims; // Array of user attributes; @@ -107,17 +107,17 @@ export class LoginProvider { claims["http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name"] || claims["http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress"]; - Logger.instanse.verbose("lookup " + username, span); + Logger.instanse.verbose("lookup " + username, span, {cls: "LoginProvider", func: "validateToken"}); const user = await Logger.DBHelper.FindByUsername(username, null, span); if (user) { - Logger.instanse.debug("succesfull", span); + Logger.instanse.debug("succesfull", span, {cls: "LoginProvider", func: "validateToken"}); resolve(user); } else { - Logger.instanse.error(new Error("Unknown user"), span); + Logger.instanse.error(new Error("Unknown user"), span, {cls: "LoginProvider", func: "validateToken"}); reject("Unknown user"); } } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "validateToken"}); reject(error); } finally { Logger.otel.endSpan(span); @@ -125,7 +125,7 @@ export class LoginProvider { }); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "validateToken"}); } finally { Logger.otel.endSpan(span); } @@ -140,16 +140,16 @@ export class LoginProvider { done(null, user._id); }); passport.deserializeUser(async function (userid: string, done: any): Promise { - Logger.instanse.silly("userid " + userid, null); + Logger.instanse.silly("userid " + userid, null, {cls: "LoginProvider", func: "deserializeUser"}); if (NoderedUtil.IsNullEmpty(userid)) return done('missing userid', null); if (typeof userid !== 'string') userid = (userid as any)._id if (NoderedUtil.IsNullEmpty(userid)) return done('missing userid', null); const _user = await Logger.DBHelper.FindById(userid, null); if (_user == null) { - Logger.instanse.error("Failed locating user " + userid, null); + Logger.instanse.error("Failed locating user " + userid, null, {cls: "LoginProvider", func: "deserializeUser"}); done(null, null); } else { - Logger.instanse.verbose("found user " + userid + " " + _user.name, null); + Logger.instanse.verbose("found user " + userid + " " + _user.name, null, {cls: "LoginProvider", func: "deserializeUser"}); done(null, _user); } // const _user = await Auth.getUser(userid, "passport"); @@ -213,7 +213,7 @@ export class LoginProvider { } if (provider.provider === "local") { hasLocal = true; } } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "RegisterProviders"}); } }); if (hasLocal === true) { @@ -226,7 +226,7 @@ export class LoginProvider { let key = keys[i]; var exists = providers.filter(x => x.id == key || (key == 'local' && x.provider == 'local')); if (exists.length == 0) { - Logger.instanse.debug("[loginprovider] Removing passport strategy " + key, span); + Logger.instanse.debug("[loginprovider] Removing passport strategy " + key, span, {cls: "LoginProvider", func: "RegisterProviders"}); passport.unuse(key); } } @@ -248,7 +248,7 @@ export class LoginProvider { static async CreateOpenIDStrategy(app: express.Express, discoveryurl: string, key: string, clientID: string, clientSecret: string, baseurl: string, span: Span): Promise { - Logger.instanse.debug("Adding new google strategy " + key, span); + Logger.instanse.debug("Adding new google strategy " + key, span, {cls: "LoginProvider", func: "CreateOpenIDStrategy"}); const response = await got.get(discoveryurl, options); const document = JSON.parse(response.body); var options = { @@ -281,15 +281,22 @@ export class LoginProvider { } } const originalUrl: any = req.cookies.originalUrl; - res.cookie("provider", key, { maxAge: 900000, httpOnly: true }); + try { + res.cookie("provider", key, { maxAge: 900000, httpOnly: true }); + } catch (error) { + } + if (!NoderedUtil.IsNullEmpty(originalUrl)) { - res.cookie("originalUrl", "", { expires: new Date(0) }); + try { + res.cookie("originalUrl", "", { expires: new Date(0) }); + } catch (error) { + } LoginProvider.redirect(res, originalUrl); } else { res.redirect("/"); } } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "CreateOpenIDStrategy"}); return res.status(500).send({ message: error.message ? error.message : error }); } finally { Logger.otel.endSpan(span); @@ -299,7 +306,7 @@ export class LoginProvider { return strategy; } static CreateGoogleStrategy(app: express.Express, key: string, consumerKey: string, consumerSecret: string, baseurl: string, span: Span): any { - Logger.instanse.debug("Adding new google strategy " + key, span); + Logger.instanse.debug("Adding new google strategy " + key, span, {cls: "LoginProvider", func: "CreateGoogleStrategy"}); const options: googleauthstrategyoptions = new googleauthstrategyoptions(); options.clientID = consumerKey; options.clientSecret = consumerSecret; @@ -323,15 +330,22 @@ export class LoginProvider { } } const originalUrl: any = req.cookies.originalUrl; - res.cookie("provider", key, { maxAge: 900000, httpOnly: true }); + try { + res.cookie("provider", key, { maxAge: 900000, httpOnly: true }); + } catch (error) { + } + if (!NoderedUtil.IsNullEmpty(originalUrl)) { - res.cookie("originalUrl", "", { expires: new Date(0) }); + try { + res.cookie("originalUrl", "", { expires: new Date(0) }); + } catch (error) { + } LoginProvider.redirect(res, originalUrl); } else { res.redirect("/"); } } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "CreateGoogleStrategy"}); return res.status(500).send({ message: error.message ? error.message : error }); } finally { Logger.otel.endSpan(span); @@ -343,7 +357,7 @@ export class LoginProvider { // tslint:disable-next-line: max-line-length static CreateSAMLStrategy(app: express.Express, key: string, cert: string, singin_url: string, issuer: string, baseurl: string, span: Span): passport.Strategy { - Logger.instanse.debug("Adding new SAML strategy " + key, span); + Logger.instanse.debug("Adding new SAML strategy " + key, span, {cls: "LoginProvider", func: "CreateSAMLStrategy"}); const options: samlauthstrategyoptions = new samlauthstrategyoptions(); (options as any).passReqToCallback = true; options.entryPoint = singin_url; @@ -416,7 +430,7 @@ export class LoginProvider { `); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "CreateSAMLStrategy"}); return res.status(500).send({ message: error.message ? error.message : error }); } finally { Logger.otel.endSpan(span); @@ -436,15 +450,22 @@ export class LoginProvider { } } const originalUrl: any = req.cookies.originalUrl; - res.cookie("provider", key, { maxAge: 900000, httpOnly: true }); + try { + res.cookie("provider", key, { maxAge: 900000, httpOnly: true }); + } catch (error) { + } + if (!NoderedUtil.IsNullEmpty(originalUrl)) { - res.cookie("originalUrl", "", { expires: new Date(0) }); + try { + res.cookie("originalUrl", "", { expires: new Date(0) }); + } catch (error) { + } LoginProvider.redirect(res, originalUrl); } else { res.redirect("/"); } } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "CreateSAMLStrategy"}); // @ts-ignore return res.status(500).send({ message: error.message ? error.message : error }); } finally { @@ -459,7 +480,7 @@ export class LoginProvider { static CreateLocalStrategy(app: express.Express, baseurl: string): passport.Strategy { const strategy: passport.Strategy = new LocalStrategy({ passReqToCallback: true }, async (req: any, username: string, password: string, done: any): Promise => { const span: Span = Logger.otel.startSpanExpress("LoginProvider.LocalLogin", req); - Logger.instanse.debug("Adding new local strategy", span); + Logger.instanse.debug("Adding new local strategy", span, {cls: "LoginProvider", func: "CreateLocalStrategy"}); try { let remoteip: string = ""; if (!NoderedUtil.IsNullUndefinded(req)) { @@ -472,7 +493,7 @@ export class LoginProvider { if (providers.length === 0 || NoderedUtil.IsNullEmpty(providers[0]._id)) { user = await Logger.DBHelper.FindByUsername(username, null, span); if (user == null) { - Logger.instanse.info("No login providers, creating " + username + " as admin", span); + Logger.instanse.info("No login providers, creating " + username + " as admin", span, {cls: "LoginProvider", func: "CreateLocalStrategy"}); user = new User(); user.name = username; user.username = username; await Crypt.SetPassword(user, password, span); const jwt: string = Crypt.rootToken(); @@ -484,24 +505,24 @@ export class LoginProvider { await Logger.DBHelper.Save(admins, Crypt.rootToken(), span) } else { if (!(await Crypt.ValidatePassword(user, password, span))) { - Logger.instanse.error("No login providers, login for " + username + " failed", span); + Logger.instanse.error("No login providers, login for " + username + " failed", span, {cls: "LoginProvider", func: "CreateLocalStrategy"}); await Audit.LoginFailed(username, "weblogin", "local", remoteip, "browser", "unknown", span); return done(null, false); } - Logger.instanse.info("No login providers, updating " + username + " as admin", span); + Logger.instanse.info("No login providers, updating " + username + " as admin", span, {cls: "LoginProvider", func: "CreateLocalStrategy"}); const admins: Role = await Logger.DBHelper.FindRoleByName("admins", null, span); if (admins == null) throw new Error("Failed locating admins role!") admins.AddMember(user); await Logger.DBHelper.Save(admins, Crypt.rootToken(), span) } - Logger.instanse.info("Clear cache", span); + Logger.instanse.info("Clear cache", span, {cls: "LoginProvider", func: "CreateLocalStrategy"}); await Logger.DBHelper.clearCache("Initialized", span); await Audit.LoginSuccess(TokenUser.From(user), "weblogin", "local", remoteip, "browser", "unknown", span); const provider: Provider = new Provider(); provider.provider = "local"; provider.name = "Local"; - Logger.instanse.info("Saving local provider", span); + Logger.instanse.info("Saving local provider", span, {cls: "LoginProvider", func: "CreateLocalStrategy"}); const result = await Config.db.InsertOne(provider, "config", 0, false, Crypt.rootToken(), span); - Logger.instanse.info("local provider created as " + result._id, span); + Logger.instanse.info("local provider created as " + result._id, span, {cls: "LoginProvider", func: "CreateLocalStrategy"}); await Logger.DBHelper.CheckCache("config", result, false, false, span); const tuser: TokenUser = TokenUser.From(user); done(null, tuser); @@ -551,7 +572,7 @@ export class LoginProvider { Logger.otel.endSpan(span); return done(null, tuser); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "CreateLocalStrategy"}); Logger.otel.endSpan(span); done(error.message ? error.message : error); } @@ -581,13 +602,13 @@ export class LoginProvider { passport.authenticate("local", function (err, user, info) { let originalUrl: any = req.cookies.originalUrl; if (err) { - Logger.instanse.error(err, null); + Logger.instanse.error(err, null, {cls: "LoginProvider", func: "Localauthenticate"}); } if (!err && user) { req.logIn(user, function (err: any) { if (err) { - Logger.instanse.debug("req.logIn failed", null); - Logger.instanse.error(err, null); + Logger.instanse.debug("req.logIn failed", null, {cls: "LoginProvider", func: "Localauthenticate"}); + Logger.instanse.error(err, null, {cls: "LoginProvider", func: "Localauthenticate"}); return next(err); } if (!NoderedUtil.IsNullEmpty(Config.validate_user_form) && req.user.validated == false) { @@ -595,12 +616,15 @@ export class LoginProvider { return next(); } else if (!NoderedUtil.IsNullEmpty(originalUrl)) { try { - res.cookie("originalUrl", "", { expires: new Date(0) }); + try { + res.cookie("originalUrl", "", { expires: new Date(0) }); + } catch (error) { + } LoginProvider.redirect(res, originalUrl); - Logger.instanse.debug("redirect: " + originalUrl, null); + Logger.instanse.debug("redirect: " + originalUrl, null, {cls: "LoginProvider", func: "Localauthenticate"}); return; } catch (error) { - Logger.instanse.error(error, null); + Logger.instanse.error(error, null, {cls: "LoginProvider", func: "Localauthenticate"}); } } else { res.redirect("/"); @@ -617,17 +641,17 @@ export class LoginProvider { } try { res.cookie("originalUrl", "", { expires: new Date(0) }); - Logger.instanse.debug("redirect: " + originalUrl, null); + Logger.instanse.debug("redirect: " + originalUrl, null, {cls: "LoginProvider", func: "Localauthenticate"}); LoginProvider.redirect(res, originalUrl); } catch (error) { - Logger.instanse.error(error, null); + Logger.instanse.error(error, null, {cls: "LoginProvider", func: "Localauthenticate"}); } } else { try { res.redirect("/"); return next(); } catch (error) { - Logger.instanse.error(error, null); + Logger.instanse.error(error, null, {cls: "LoginProvider", func: "Localauthenticate"}); } } })(req, res, next); @@ -643,7 +667,7 @@ export class LoginProvider { let username: string = profile.username; if (NoderedUtil.IsNullEmpty(username)) username = profile.nameID; if (!NoderedUtil.IsNullEmpty(username)) { username = username.toLowerCase(); } - Logger.instanse.debug(username, span); + Logger.instanse.debug(username, span, {cls: "LoginProvider", func: "samlverify"}); let _user: User = await Logger.DBHelper.FindByUsernameOrFederationid(username, issuer, span); let remoteip: string = ""; if (!NoderedUtil.IsNullUndefinded(req)) { @@ -722,7 +746,7 @@ export class LoginProvider { Logger.otel.endSpan(span); done(null, tuser); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "samlverify"}); } Logger.otel.endSpan(span); } @@ -739,7 +763,7 @@ export class LoginProvider { let username: string = profile.username; if (NoderedUtil.IsNullEmpty(username)) username = profile.nameID; if (!NoderedUtil.IsNullEmpty(username)) { username = username.toLowerCase(); } - Logger.instanse.debug(profile.id, span); + Logger.instanse.debug(profile.id, span, {cls: "LoginProvider", func: "openidverify"}); let _user: User = await Logger.DBHelper.FindByUsernameOrFederationid(profile.id, issuer, span); if (NoderedUtil.IsNullUndefinded(_user)) { _user = await Logger.DBHelper.FindByUsernameOrFederationid(profile.username, issuer, span); @@ -788,7 +812,7 @@ export class LoginProvider { await Audit.LoginSuccess(tuser, "weblogin", "openid", remoteip, "openidverify" as any, "unknown", span); done(null, tuser); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "openidverify"}); } Logger.otel.endSpan(span); } @@ -808,7 +832,7 @@ export class LoginProvider { let username: string = profile.username; if (NoderedUtil.IsNullEmpty(username)) username = profile.nameID; if (!NoderedUtil.IsNullEmpty(username)) { username = username.toLowerCase(); } - Logger.instanse.debug(username, span); + Logger.instanse.debug(username, span, {cls: "LoginProvider", func: "googleverify"}); let _user: User = await Logger.DBHelper.FindByUsernameOrFederationid(username, issuer, span); if (NoderedUtil.IsNullUndefinded(_user)) { let createUser: boolean = Config.auto_create_users; @@ -852,7 +876,7 @@ export class LoginProvider { await Audit.LoginSuccess(tuser, "weblogin", "google", remoteip, "unknown", "unknown", span); done(null, tuser); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "googleverify"}); } Logger.otel.endSpan(span); } @@ -891,8 +915,8 @@ export class LoginProvider { let from = Config.smtp_from; if (Config.NODE_ENV != "production") { - Logger.instanse.warn("Skip sending email to " + to, span); - Logger.instanse.info(text, span); + Logger.instanse.warn("Skip sending email to " + to, span, {cls: "LoginProvider", func: "sendEmail"}); + Logger.instanse.info(text, span, {cls: "LoginProvider", func: "sendEmail"}); resolve("email not sent"); } else { transporter.sendMail({ @@ -902,10 +926,10 @@ export class LoginProvider { html }, function (error, info) { if (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "sendEmail"}); reject(error); } else { - Logger.instanse.info("Email sent to " + to + " " + info.response, span); + Logger.instanse.info("Email sent to " + to + " " + info.response, span, {cls: "LoginProvider", func: "sendEmail"}); var item: any = new Base(); item.readcount = 0; item._type = type; @@ -930,23 +954,23 @@ export class LoginProvider { try { span?.setAttribute("remoteip", LoginProvider.remoteip(req)); if (req.user) { - Logger.instanse.verbose("User is signed in", span); + Logger.instanse.verbose("User is signed in", span, {cls: "LoginProvider", func: "dashboardauth"}); const user: TokenUser = TokenUser.From(req.user); span?.setAttribute("username", user.username); if (user != null) { const allowed = user.roles.filter(x => x.name == "dashboardusers" || x.name == "admins"); if (allowed.length > 0) { - Logger.instanse.verbose("Authorized " + user.username + " for " + req.url, span); + Logger.instanse.verbose("Authorized " + user.username + " for " + req.url, span, {cls: "LoginProvider", func: "dashboardauth"}); return res.send({ status: "success", display_status: "Success", message: "Connection OK" }); } else { - Logger.instanse.warn((user.username + " is not member of 'dashboardusers' for " + req.url), span); + Logger.instanse.warn((user.username + " is not member of 'dashboardusers' for " + req.url), span, {cls: "LoginProvider", func: "dashboardauth"}); } } else { - Logger.instanse.error("Failed casting user", span); + Logger.instanse.error("Failed casting user", span, {cls: "LoginProvider", func: "dashboardauth"}); } } const authorization: string = req.headers.authorization; @@ -958,19 +982,19 @@ export class LoginProvider { return; } - Logger.instanse.verbose("Lookup user by authentication header", span); + Logger.instanse.verbose("Lookup user by authentication header", span, {cls: "LoginProvider", func: "dashboardauth"}); var user: User = await Logger.DBHelper.FindByAuthorization(authorization, null, span); if (user != null) { const allowed = user.roles.filter(x => x.name == "dashboardusers" || x.name == "admins"); if (allowed.length > 0) { - Logger.instanse.debug("User is authorized to see dashboard", span); + Logger.instanse.debug("User is authorized to see dashboard", span, {cls: "LoginProvider", func: "dashboardauth"}); return res.send({ status: "success", display_status: "Success", message: "Connection OK" }); } else { - Logger.instanse.warn(user.username + " is not member of 'dashboardusers' for " + req.url, span); + Logger.instanse.warn(user.username + " is not member of 'dashboardusers' for " + req.url, span, {cls: "LoginProvider", func: "dashboardauth"}); } } res.statusCode = 401; @@ -994,15 +1018,15 @@ export class LoginProvider { res.cookie("_session", "", { expires: new Date(0) }); const originalUrl: any = req.cookies.originalUrl; if (!NoderedUtil.IsNullEmpty(originalUrl)) { - Logger.instanse.debug("Redirect user to " + originalUrl, span); + Logger.instanse.debug("Redirect user to " + originalUrl, span, {cls: "LoginProvider", func: "Signout"}); res.cookie("originalUrl", "", { expires: new Date(0) }); LoginProvider.redirect(res, originalUrl); } else { - Logger.instanse.debug("Redirect user to /", span); + Logger.instanse.debug("Redirect user to /", span, {cls: "LoginProvider", func: "Signout"}); res.redirect("/"); } } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "Signout"}); return res.status(500).send({ message: error.message ? error.message : error }); } finally { Logger.otel.endSpan(span); @@ -1014,15 +1038,15 @@ export class LoginProvider { req.logout(); const originalUrl: any = req.cookies.originalUrl; if (!NoderedUtil.IsNullEmpty(originalUrl)) { - Logger.instanse.debug("Redirect user to " + originalUrl, span); + Logger.instanse.debug("Redirect user to " + originalUrl, span, {cls: "LoginProvider", func: "PassiveSignout"}); res.cookie("originalUrl", "", { expires: new Date(0) }); LoginProvider.redirect(res, originalUrl); } else { - Logger.instanse.debug("Redirect user to /", span); + Logger.instanse.debug("Redirect user to /", span, {cls: "LoginProvider", func: "PassiveSignout"}); res.redirect("/Login"); } } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "PassiveSignout"}); return res.status(500).send({ message: error.message ? error.message : error }); } finally { Logger.otel.endSpan(span); @@ -1034,7 +1058,7 @@ export class LoginProvider { span?.setAttribute("remoteip", LoginProvider.remoteip(req)); res.setHeader("Content-Type", "application/json"); if (req.user) { - Logger.instanse.debug("return user " + req.user._id, span); + Logger.instanse.debug("return user " + req.user._id, span, {cls: "LoginProvider", func: "getuser"}); await Logger.DBHelper.UserRoleUpdateId(req.user._id, false, span); const user: User = await Logger.DBHelper.FindById(req.user._id, span); user.validated = true; @@ -1046,7 +1070,7 @@ export class LoginProvider { } res.end(JSON.stringify(user)); } else { - Logger.instanse.debug("return nothing, not signed in", span); + Logger.instanse.debug("return nothing, not signed in", span, {cls: "LoginProvider", func: "getuser"}); res.end(JSON.stringify({})); } res.end(); @@ -1062,15 +1086,15 @@ export class LoginProvider { if (req.user) { const user: TokenUser = TokenUser.From(req.user); span?.setAttribute("username", user.username); - Logger.instanse.debug("return token for user " + req.user._id + " " + user.name, span); + Logger.instanse.debug("return token for user " + req.user._id + " " + user.name, span, {cls: "LoginProvider", func: "getjwt"}); res.end(JSON.stringify({ jwt: Crypt.createToken(user, Config.shorttoken_expires_in), user: user })); } else { - Logger.instanse.verbose("return nothing, not signed in", span); + Logger.instanse.verbose("return nothing, not signed in", span, {cls: "LoginProvider", func: "getjwt"}); res.end(JSON.stringify({ jwt: "" })); } res.end(); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "getjwt"}); return res.status(500).send({ message: error.message ? error.message : error }); } finally { Logger.otel.endSpan(span); @@ -1086,19 +1110,19 @@ export class LoginProvider { span?.setAttribute("username", user.username); if (!(user.validated == true) && Config.validate_user_form != "") { - Logger.instanse.error("return nothing, user is not validated yet", span); + Logger.instanse.error("return nothing, user is not validated yet", span, {cls: "LoginProvider", func: "getjwtlong"}); res.end(JSON.stringify({ jwt: "" })); } else { - Logger.instanse.debug("return token for user " + req.user._id + " " + user.name, span); + Logger.instanse.debug("return token for user " + req.user._id + " " + user.name, span, {cls: "LoginProvider", func: "getjwtlong"}); res.end(JSON.stringify({ jwt: Crypt.createToken(user, Config.longtoken_expires_in), user: user })); } } else { - Logger.instanse.error("return nothing, not signed in", span); + Logger.instanse.error("return nothing, not signed in", span, {cls: "LoginProvider", func: "getjwtlong"}); res.end(JSON.stringify({ jwt: "" })); } res.end(); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "getjwtlong"}); return res.status(500).send({ message: error.message ? error.message : error }); } finally { Logger.otel.endSpan(span); @@ -1114,10 +1138,10 @@ export class LoginProvider { const tuser: TokenUser = TokenUser.From(user); span?.setAttribute("username", user.username); res.setHeader("Content-Type", "application/json"); - Logger.instanse.debug("Recreating jwt token", span); + Logger.instanse.debug("Recreating jwt token", span, {cls: "LoginProvider", func: "postjwt"}); res.end(JSON.stringify({ jwt: Crypt.createToken(tuser, Config.shorttoken_expires_in) })); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "postjwt"}); return res.status(500).send({ message: error.message ? error.message : error }); } finally { Logger.otel.endSpan(span); @@ -1168,7 +1192,8 @@ export class LoginProvider { ping_clients_interval: Config.ping_clients_interval, validlicense: Logger.License.validlicense, forceddomains: forceddomains, - grafana_url: Config.grafana_url + grafana_url: Config.grafana_url, + llmchat_queue: Config.llmchat_queue } return res2; } @@ -1181,10 +1206,10 @@ export class LoginProvider { span?.setAttribute("username", user.username); } const res2 = await LoginProvider.config(); - Logger.instanse.debug("Return configuration settings", span); + Logger.instanse.debug("Return configuration settings", span, {cls: "LoginProvider", func: "getconfig"}); res.end(JSON.stringify(res2)); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "getconfig"}); return res.status(500).send({ message: error.message ? error.message : error }); } finally { Logger.otel.endSpan(span); @@ -1198,14 +1223,14 @@ export class LoginProvider { const key = req.body.key; let exists: TokenRequest = await Logger.DBHelper.FindRequestTokenID(key, span); if (!NoderedUtil.IsNullUndefinded(exists)) { - Logger.instanse.error("Key has already been used! " + key, span); + Logger.instanse.error("Key has already been used! " + key, span, {cls: "LoginProvider", func: "AddTokenRequest"}); return res.status(500).send({ message: "Illegal key" }); } await Logger.DBHelper.AddRequestTokenID(key, {}, span); - Logger.instanse.info("Added token request " + key + " from " + remoteip, span); + Logger.instanse.info("Added token request " + key + " from " + remoteip, span, {cls: "LoginProvider", func: "AddTokenRequest"}); res.status(200).send({ message: "ok" }); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "AddTokenRequest"}); return res.status(500).send({ message: error.message ? error.message : error }); } finally { Logger.otel.endSpan(span); @@ -1214,44 +1239,45 @@ export class LoginProvider { static async get_GetTokenRequest(req: any, res: any, next: any): Promise { const span: Span = Logger.otel.startSpanExpress("LoginProvider.login", req); try { - span?.setAttribute("remoteip", LoginProvider.remoteip(req)); + const remoteip = LoginProvider.remoteip(req); + span?.setAttribute("remoteip", remoteip); const key = req.query.key; let exists: TokenRequest = null; exists = await Logger.DBHelper.FindRequestTokenID(key, span); if (NoderedUtil.IsNullUndefinded(exists)) { - Logger.instanse.error("Unknown key " + key, span); + Logger.instanse.error("Unknown key " + key + " from " + remoteip, span, {remoteip, cls: "LoginProvider", func: "GetTokenRequest"}); res.status(200).send({ message: "Illegal key" }); return; } if (!NoderedUtil.IsNullEmpty(exists.jwt)) { - Logger.instanse.info("Token " + key + " has been forfilled", span); + Logger.instanse.info("Token " + key + " has been forfilled from " + remoteip, span, {remoteip, cls: "LoginProvider", func: "GetTokenRequest"}); if (Config.validate_user_form != "") { try { var tuser = await await Crypt.verityToken(exists.jwt); var user = await Logger.DBHelper.FindById(tuser._id, span); if (user.validated == true) { await Logger.DBHelper.RemoveRequestTokenID(key, span); - Logger.instanse.debug("return jwt for " + key, span); + Logger.instanse.debug("return jwt for " + key, span, {remoteip, cls: "LoginProvider", func: "GetTokenRequest"}); res.status(200).send(Object.assign(exists, { message: "ok" })); } else { - Logger.instanse.debug("User not validated yet, for key " + key + " user " + user.name + " " + user._id, span); + Logger.instanse.debug("User not validated yet, for key " + key + " user " + user.name + " " + user._id, span, {remoteip, cls: "LoginProvider", func: "GetTokenRequest"}); res.status(200).send({ message: "ok" }); } } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {remoteip, cls: "LoginProvider", func: "GetTokenRequest"}); } } else { - Logger.instanse.debug("return jwt for " + key, span); + Logger.instanse.debug("return jwt for " + key, span, {remoteip, cls: "LoginProvider", func: "GetTokenRequest"}); res.status(200).send(Object.assign(exists, { message: "ok" })); await Logger.DBHelper.RemoveRequestTokenID(key, span); } } else { - Logger.instanse.debug("No jwt for " + key, span); + Logger.instanse.debug("No jwt for " + key, span, {remoteip, cls: "LoginProvider", func: "GetTokenRequest"}); res.status(200).send(Object.assign(exists, { message: "ok" })); } } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "GetTokenRequest"}); try { res.status(500).send({ message: error.message ? error.message : error }); } catch (error) { @@ -1276,25 +1302,31 @@ export class LoginProvider { const user: User = await Logger.DBHelper.FindById(req.user._id, span); var exists: TokenRequest = await Logger.DBHelper.FindRequestTokenID(key, span); if (!NoderedUtil.IsNullUndefinded(exists)) { - Logger.instanse.debug("adding jwt for request token " + key, span); + Logger.instanse.debug("adding jwt for request token " + key, span, {cls: "LoginProvider", func: "getlogin"}); await Logger.DBHelper.AddRequestTokenID(key, { jwt: Crypt.createToken(user, Config.longtoken_expires_in) }, span); - res.cookie("requesttoken", "", { expires: new Date(0) }); + try { + res.cookie("requesttoken", "", { expires: new Date(0) }); + } catch (error) { + } } } else { - res.cookie("requesttoken", key, { maxAge: 36000, httpOnly: true }); + try { + res.cookie("requesttoken", key, { maxAge: 36000, httpOnly: true }); + } catch (error) { + } } } if (!NoderedUtil.IsNullEmpty(req.query.key)) { if (req.user) { res.cookie("originalUrl", "", { expires: new Date(0) }); - Logger.instanse.debug("User signed in, with key " + key, span); + Logger.instanse.debug("User signed in, with key " + key, span, {cls: "LoginProvider", func: "getlogin"}); this.redirect(res, "/"); } else { try { res.cookie("originalUrl", req.originalUrl, { maxAge: 900000, httpOnly: true }); } catch (error) { } - Logger.instanse.debug("User not signed in, redirect to /login", span); + Logger.instanse.debug("User not signed in, redirect to /login", span, {cls: "LoginProvider", func: "getlogin"}); this.redirect(res, "/login"); } } @@ -1315,8 +1347,11 @@ export class LoginProvider { const originalUrl: any = req.cookies.originalUrl; const validateurl: any = req.cookies.validateurl; if (NoderedUtil.IsNullEmpty(originalUrl) && !req.originalUrl.startsWith("/login")) { - Logger.instanse.debug("Save originalUrl as " + originalUrl, span); - res.cookie("originalUrl", req.originalUrl, { maxAge: 900000, httpOnly: true }); + Logger.instanse.debug("Save originalUrl as " + originalUrl, span, {cls: "LoginProvider", func: "getlogin"}); + try { + res.cookie("originalUrl", req.originalUrl, { maxAge: 900000, httpOnly: true }); + } catch (error) { + } } if (!NoderedUtil.IsNullEmpty(validateurl)) { if (tuser != null) { @@ -1327,7 +1362,7 @@ export class LoginProvider { } else { res.cookie("validateurl", "", { expires: new Date(0) }); res.cookie("originalUrl", "", { expires: new Date(0) }); - Logger.instanse.debug("redirect to validateurl /#" + validateurl, span); + Logger.instanse.debug("redirect to validateurl /#" + validateurl, span, {cls: "LoginProvider", func: "getlogin"}); this.redirect(res, "/#" + validateurl); return; } @@ -1336,19 +1371,19 @@ export class LoginProvider { if (req.user != null && !NoderedUtil.IsNullEmpty(originalUrl) && tuser.validated) { if (!NoderedUtil.IsNullEmpty(Config.validate_user_form) && req.user.validated == true) { if(originalUrl != "/login" && originalUrl != "/Login") { - Logger.instanse.debug("user validated, redirect to " + originalUrl, span); + Logger.instanse.debug("user validated, redirect to " + originalUrl, span, {cls: "LoginProvider", func: "getlogin"}); this.redirect(res, originalUrl); } else { - Logger.instanse.debug("user signed in, redirect to /", span); + Logger.instanse.debug("user signed in, redirect to /", span, {cls: "LoginProvider", func: "getlogin"}); this.redirect(res, "/"); } return; } else if (NoderedUtil.IsNullEmpty(Config.validate_user_form)) { if(originalUrl != "/login" && originalUrl != "/Login") { - Logger.instanse.debug("user signed in, redirect to " + originalUrl, span); + Logger.instanse.debug("user signed in, redirect to " + originalUrl, span, {cls: "LoginProvider", func: "getlogin"}); this.redirect(res, originalUrl); } else { - Logger.instanse.debug("user signed in, redirect to /", span); + Logger.instanse.debug("user signed in, redirect to /", span, {cls: "LoginProvider", func: "getlogin"}); this.redirect(res, "/"); } return; @@ -1358,15 +1393,15 @@ export class LoginProvider { } } if (tuser != null && tuser.validated) { - Logger.instanse.debug("redirect to /", span); + Logger.instanse.debug("redirect to /", span, {cls: "LoginProvider", func: "getlogin"}); this.redirect(res, "/"); } else { - Logger.instanse.debug("return PassiveLogin.html", span); + Logger.instanse.debug("return PassiveLogin.html", span, {cls: "LoginProvider", func: "getlogin"}); const file = path.join(__dirname, 'public', 'PassiveLogin.html'); res.sendFile(file); } } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "getlogin"}); try { return res.status(500).send({ message: error.message ? error.message : error }); } catch (error) { @@ -1380,7 +1415,7 @@ export class LoginProvider { span?.setAttribute("remoteip", LoginProvider.remoteip(req)); res.setHeader("Content-Type", "application/json"); if (NoderedUtil.IsNullEmpty(Config.validate_user_form)) { - Logger.instanse.debug("No validate user form set, return nothing", span); + Logger.instanse.debug("No validate user form set, return nothing", span, {cls: "LoginProvider", func: "validateuserform"}); res.end(JSON.stringify({})); res.end(); Logger.otel.endSpan(span); @@ -1388,18 +1423,18 @@ export class LoginProvider { } var forms = await Config.db.query({ query: { _id: Config.validate_user_form, _type: "form" }, top: 1, collectionname: "forms", jwt: Crypt.rootToken() }, span); if (forms.length == 1) { - Logger.instanse.debug("Return form " + Config.validate_user_form, span); + Logger.instanse.debug("Return form " + Config.validate_user_form, span, {cls: "LoginProvider", func: "validateuserform"}); res.end(JSON.stringify(forms[0])); res.end(); Logger.otel.endSpan(span); return; } - Logger.instanse.error("validate_user_form " + Config.validate_user_form + " does not exists!", span); + Logger.instanse.error("validate_user_form " + Config.validate_user_form + " does not exists!", span, {cls: "LoginProvider", func: "validateuserform"}); Config.validate_user_form = ""; res.end(JSON.stringify({})); res.end(); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "validateuserform"}); return res.status(500).send({ message: error.message ? error.message : error }); } finally { Logger.otel.endSpan(span); @@ -1425,7 +1460,7 @@ export class LoginProvider { const UpdateDoc: any = { "$set": { "_modified": dt, "read": true }, "$push": { "opened": { dt, ip, domain, agent } }, "$inc": { "readcount": 1 } }; var res2 = await Config.db._UpdateOne({ id }, UpdateDoc, "mailhist", 1, true, Crypt.rootToken(), null); } catch (error) { - Logger.instanse.error(error, null); + Logger.instanse.error(error, null, {cls: "LoginProvider", func: "read"}); } } static async post_validateuserform(req: any, res) { @@ -1464,14 +1499,14 @@ export class LoginProvider { if (Config.validate_emails) { if (Config.smtp_service == "gmail") { if (NoderedUtil.IsNullEmpty(Config.smtp_user) || NoderedUtil.IsNullEmpty(Config.smtp_pass)) { - Logger.instanse.error("Disabling email validation, missing login information fot gmail", span); + Logger.instanse.error("Disabling email validation, missing login information fot gmail", span, {cls: "LoginProvider", func: "validateuserform"}); Config.validate_emails = false; } } else if (NoderedUtil.IsNullEmpty(Config.smtp_url)) { - Logger.instanse.error("Disabling email validation, missing smtp_url", span); + Logger.instanse.error("Disabling email validation, missing smtp_url", span, {cls: "LoginProvider", func: "validateuserform"}); Config.validate_emails = false; } else if (NoderedUtil.IsNullEmpty(Config.smtp_from)) { - Logger.instanse.error("Disabling email validation, missing smtp_from", span); + Logger.instanse.error("Disabling email validation, missing smtp_from", span, {cls: "LoginProvider", func: "validateuserform"}); Config.validate_emails = false; } } @@ -1501,9 +1536,10 @@ export class LoginProvider { // https://disposable.debounce.io/?email=info@example.com email = email.toLowerCase(); - var exists = await Config.db.query({ query: { "_id": { "$ne": tuser._id }, "$or": [{ "username": email }, { "email": email }], "_type": "user" }, collectionname: "users", jwt: Crypt.rootToken() }, span); + var exists = await Config.db.query({ query: { "$or": [{ "username": email }, { "email": email }], "_type": "user" }, collectionname: "users", jwt: Crypt.rootToken() }, span); + exists = exists.filter(x => x._id != tuser._id); if (exists.length > 0) { - Logger.instanse.error(tuser.name + " trying to register email " + email + " already used by " + exists[0].name + " (" + exists[0]._id + ")", span) + Logger.instanse.error(tuser.name + " trying to register email " + email + " already used by " + exists[0].name + " (" + exists[0]._id + ")", span, {cls: "LoginProvider", func: "validateuserform"}) email = ""; delete UpdateDoc.$set["email"]; UpdateDoc.$set["formvalidated"] = false; @@ -1521,7 +1557,7 @@ export class LoginProvider { `Hi ${tuser.name}\nPlease use the below code to validate your email\n${code}`, span); } } else { - Logger.instanse.error(tuser.name + " email is mandatory)", span); + Logger.instanse.error(tuser.name + " email is mandatory)", span, {cls: "LoginProvider", func: "validateuserform"}); throw new Error("email is mandatory."); } } else { @@ -1530,15 +1566,15 @@ export class LoginProvider { } - Logger.instanse.debug("Update user " + tuser.name + " information", span); + Logger.instanse.debug("Update user " + tuser.name + " information", span, {cls: "LoginProvider", func: "validateuserform"}); var res2 = await Config.db._UpdateOne({ "_id": tuser._id }, UpdateDoc, "users", 1, true, Crypt.rootToken(), span); await Logger.DBHelper.CheckCache("users", tuser as any, false, false, span); } if (!(tuser.validated == true) && Config.validate_user_form != "") { - Logger.instanse.debug("User not validated, return no token for user " + tuser.name, span); + Logger.instanse.debug("User not validated, return no token for user " + tuser.name, span, {cls: "LoginProvider", func: "validateuserform"}); res.end(JSON.stringify({ jwt: "" })); } else { - Logger.instanse.debug("Return new jwt for user " + tuser.name, span); + Logger.instanse.debug("Return new jwt for user " + tuser.name, span, {cls: "LoginProvider", func: "validateuserform"}); res.end(JSON.stringify({ jwt: Crypt.createToken(tuser, Config.longtoken_expires_in), user: tuser })); } } else if (req.body) { @@ -1591,11 +1627,11 @@ export class LoginProvider { } } else { - Logger.instanse.error("User no longer signed in", span); + Logger.instanse.error("User no longer signed in", span, {cls: "LoginProvider", func: "validateuserform"}); res.end(JSON.stringify({ jwt: "" })); } } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "validateuserform"}); return res.status(500).send({ message: error.message ? error.message : error }); } Logger.otel.endSpan(span); @@ -1610,17 +1646,17 @@ export class LoginProvider { const email: string = req.body.email; let user = await Config.db.getbyusername(req.body.email, null, Crypt.rootToken(), true, span); if (user == null) { - Logger.instanse.error("Received unknown email " + email, span); + Logger.instanse.error("Received unknown email " + email, span, {cls: "LoginProvider", func: "forgotpassword"}); return res.end(JSON.stringify({ id })); } const code = NoderedUtil.GetUniqueIdentifier(); var key = ("forgotpass_" + id).toString(); let item = await Logger.DBHelper.memoryCache.wrap(key, () => { - Logger.instanse.info(`Add forgotpass if ${id} with code ${code} for ${email}`, span); + Logger.instanse.info(`Add forgotpass if ${id} with code ${code} for ${email}`, span, {cls: "LoginProvider", func: "forgotpassword"}); return { id, email, code }; }); if (item.id != id || item.email != email || item.code != code) { - Logger.instanse.error("Recevied wrong mail for id " + id, span); + Logger.instanse.error("Recevied wrong mail for id " + id, span, {cls: "LoginProvider", func: "forgotpassword"}); return res.end(JSON.stringify({ id })); } this.sendEmail("pwreset", user._id, email, 'Reset password request', @@ -1638,9 +1674,9 @@ export class LoginProvider { }); if (item == null || item.id != id || item.code != code) { if (item == null) { - Logger.instanse.error("Recevied unknown id " + id, span); + Logger.instanse.error("Recevied unknown id " + id, span, {cls: "LoginProvider", func: "forgotpassword"}); } else { - Logger.instanse.error("Recevied wrong code for id " + id, span); + Logger.instanse.error("Recevied wrong code for id " + id, span, {cls: "LoginProvider", func: "forgotpassword"}); } throw new Error("Recevied wrong code for id " + id); } @@ -1653,11 +1689,11 @@ export class LoginProvider { var key = ("forgotpass_" + id).toString(); let item = await Logger.DBHelper.memoryCache.wrap(key, () => { - Logger.instanse.debug("Add forgotpass : " + id, span); + Logger.instanse.debug("Add forgotpass : " + id, span, {cls: "LoginProvider", func: "forgotpassword"}); return null; }); if (item == null || item.id != id || item.code != code) { - Logger.instanse.error("Recevied wrong code for id " + id, span); + Logger.instanse.error("Recevied wrong code for id " + id, span, {cls: "LoginProvider", func: "forgotpassword"}); throw new Error("Recevied wrong code for id " + id); } let user = await Config.db.getbyusername(item.email, null, Crypt.rootToken(), true, span); @@ -1676,7 +1712,7 @@ export class LoginProvider { } res.end(JSON.stringify({})); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "forgotpassword"}); return res.status(500).send({ message: error.message ? error.message : error }); } Logger.otel.endSpan(span); @@ -1691,7 +1727,7 @@ export class LoginProvider { res.end(JSON.stringify(result)); res.end(); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "loginproviders"}); Logger.otel.endSpan(span); return res.status(500).send({ message: error.message ? error.message : error }); } finally { @@ -1732,7 +1768,7 @@ export class LoginProvider { }); downloadStream.pipe(res); } catch (error) { - Logger.instanse.error(error, span); + Logger.instanse.error(error, span, {cls: "LoginProvider", func: "download"}); return res.status(500).send({ message: error.message ? error.message : error }); } finally { Logger.otel.endSpan(span); diff --git a/OpenFlow/src/Messages/Message.ts b/OpenFlow/src/Messages/Message.ts index aa43a185..7bb0930f 100644 --- a/OpenFlow/src/Messages/Message.ts +++ b/OpenFlow/src/Messages/Message.ts @@ -197,12 +197,14 @@ export class Message { public static async verityToken(token:string, cli?: WebSocketServerClient, ) { if(token == null || token == "") return null; var user = null; - try { - user = User.assign(await Crypt.verityToken(token, cli)); - if(user!=null) return user; - } catch (error) { - } var AccessToken = await OAuthProvider.instance.oidc.AccessToken.find(token); + if (NoderedUtil.IsNullUndefinded(AccessToken)) { + try { + user = User.assign(await Crypt.verityToken(token, cli)); + if(user!=null) return user; + } catch (error) { + } + } if (!NoderedUtil.IsNullUndefinded(AccessToken)) { user = await OAuthProvider.instance.oidc.Account.findAccount(null, AccessToken.accountId); } else { @@ -621,6 +623,12 @@ export class Message { case "createindex": await this.CreateIndex(span); break; + case "dropindex": + await this.DropIndex(span); + break; + case "getindexes": + await this.GetIndexes(span); + break; case "deletepackage": await this.DeletePackage(span); break; @@ -730,6 +738,35 @@ export class Message { this.data = JSON.stringify(msg); } } + async GetIndexes(parent: Span) { + this.Reply(); + let msg: any = this.data + if( typeof this.data == "string") { + msg = JSON.stringify(this.data) + } + msg = JSON.parse(JSON.stringify(msg)); + try { + if (!this.tuser.HasRoleId(WellknownIds.admins)) throw new Error("Access denied"); + const indexes = await Config.db.db.collection(msg.collectionname).indexes(); + msg.results = indexes; + } finally { + this.data = JSON.stringify(msg); + } + } + async DropIndex(parent: Span) { + this.Reply(); + let msg: any = this.data + if( typeof this.data == "string") { + msg = JSON.stringify(this.data) + } + msg = JSON.parse(JSON.stringify(msg)); + try { + if (!this.tuser.HasRoleId(WellknownIds.admins)) throw new Error("Access denied"); + msg.result = await Config.db.deleteIndex(msg.collectionname, msg.name, parent); + } finally { + this.data = JSON.stringify(msg); + } + } async DeletePackage(parent: Span) { this.Reply(); let msg: any = this.data @@ -1180,7 +1217,7 @@ export class Message { msg.result = Message.collectionCache[msg.jwt]; } else { span?.addEvent("ListCollections"); - msg.result = await Config.db.ListCollections(msg.jwt); + msg.result = await Config.db.ListCollections(false, msg.jwt); span?.addEvent("Filter collections"); if (msg.includehist !== true) { msg.result = msg.result.filter(x => !x.name.endsWith("_hist")); @@ -1353,7 +1390,8 @@ export class Message { if (NoderedUtil.IsNullEmpty(msg.jwt)) { msg.jwt = this.jwt; } // @ts-ignore var queryas = msg.queryas; - msg.result = await Config.db.aggregate(msg.aggregates, msg.collectionname, msg.jwt, msg.hint, queryas, span); + // @ts-ignore + msg.result = await Config.db.aggregate(msg.aggregates, msg.collectionname, msg.jwt, msg.hint, queryas, msg.explain, span); if (this.clientagent == "openrpa") Config.db.parseResults(msg.result, this.clientagent, this.clientversion); delete msg.aggregates; delete msg.jwt; @@ -2876,12 +2914,19 @@ export class Message { // Add requested quantity, now we have our target count _quantity += quantity; - const stripe_product = await this.Stripe("GET", "products", product.stripeproduct, null, null); - if(stripe_product==null) throw new Error("Unknown product"); - if(stripe_product.active == false) throw new Error("Product is not active"); - const stripe_price = await this.Stripe("GET", "prices", product.stripeprice, null, null); - if(stripe_price==null) throw new Error("Unknown price " + product.stripeprice + " for product " + product.name); - if(stripe_price.active == false) throw new Error("Price " + product.stripeprice + " for product " + product.name + " is not active"); + if(!NoderedUtil.IsNullEmpty(product.stripeproduct) && !NoderedUtil.IsNullEmpty(Config.stripe_api_secret)) { + const stripe_product = await this.Stripe("GET", "products", product.stripeproduct, null, null); + if(stripe_product==null) throw new Error("Unknown product"); + if(stripe_product.active == false) throw new Error("Product is not active"); + } + let stripe_price: stripe_price = {type: "payment"} as any; + + if(!NoderedUtil.IsNullEmpty(product.stripeprice) && !NoderedUtil.IsNullEmpty(Config.stripe_api_secret)) { + stripe_price = await this.Stripe("GET", "prices", product.stripeprice, null, null); + if(stripe_price==null) throw new Error("Unknown price " + product.stripeprice + " for product " + product.name); + if(stripe_price.active == false) throw new Error("Price " + product.stripeprice + " for product " + product.name + " is not active"); + } + if((stripe_price as any).type != "one_time"){ if (NoderedUtil.IsNullEmpty(usage.subid)) { @@ -4018,7 +4063,7 @@ export class Message { const user = Crypt.rootUser(); const tuser = TokenUser.From(user); const jwt: string = Crypt.rootToken(); - let collections = await Config.db.ListCollections(jwt); + let collections = await Config.db.ListCollections(false, jwt); collections = collections.filter(x => x.name.indexOf("system.") === -1); let totalusage = 0; let index = 0; @@ -5141,6 +5186,14 @@ export class Message { const rootjwt = Crypt.rootToken(); const jwt = this.jwt; msg = AddWorkitemQueueMessage.assign(this.data); + + + var skiprole = msg.skiprole; + // @ts-ignore + if(this.data.skiprole != null) { + // @ts-ignore + skiprole = this.data.skiprole; + } // @ts-ignore if(this.data.workitemqueue != null) msg = this.data.workitemqueue; if (NoderedUtil.IsNullEmpty(msg.name)) throw new Error("Name is mandatory") @@ -5156,7 +5209,7 @@ export class Message { var wiq = new WorkitemQueue(); wiq._type = "workitemqueue"; const workitem_queue_admins: Role = await Logger.DBHelper.EnsureRole(jwt, "workitem queue admins", "625440c4231309af5f2052cd", parent); - if (!msg.skiprole) { + if (!skiprole) { const wiqusers: Role = await Logger.DBHelper.EnsureRole(jwt, msg.name + " users", null, parent); Base.addRight(wiqusers, WellknownIds.admins, "admins", [Rights.full_control]); Base.addRight(wiqusers, user._id, user.name, [Rights.full_control]); @@ -5177,6 +5230,10 @@ export class Message { wiq.projectid = msg.projectid; wiq.amqpqueue = msg.amqpqueue; wiq.maxretries = msg.maxretries; + if(msg._acl != null) { + // @ts-ignore + wiq._acl = JSON.parse(JSON.stringify(msg._acl)); + } if(wiq.maxretries < 1) wiq.maxretries = 3; wiq.retrydelay = msg.retrydelay; wiq.initialdelay = msg.initialdelay; diff --git a/OpenFlow/src/OAuthProvider.ts b/OpenFlow/src/OAuthProvider.ts index 968f9bc4..1a2762bb 100644 --- a/OpenFlow/src/OAuthProvider.ts +++ b/OpenFlow/src/OAuthProvider.ts @@ -178,6 +178,24 @@ export class OAuthProvider { } ) } + var agent = instance.clients.find(x => x.client_id == "openapi"); + if(agent == null) { + // token_endpoint_auth_method can only be none, client_secret_post, client_secret_basic, private_key_jwt or tls_client_auth + instance.clients.push({ + grants: ['password', 'refresh_token', 'authorization_code'], + defaultrole : "Viewer", + rolemappings : { "admins": "Admin" }, + clientId: "openapi",client_id: "openapi", + client_secret: "openapi", + token_endpoint_auth_method: "client_secret_post", + response_types: ['code', 'id_token', 'code id_token'], + grant_types: ['implicit', 'authorization_code'], + post_logout_redirect_uris: [], + redirect_uris: [], + openflowsignout: true + } + ) + } // var grafana = instance.clients.find(x => x.client_id == "grafana"); // if(grafana == null) { // instance.clients.push({ @@ -289,6 +307,11 @@ export class OAuthProvider { // if (req.originalUrl.indexOf('/oidc') > -1) return next(); }); + instance.app.use('/oidc/*', async (req, res, next) => { + console.log(req); + next(); + }); + instance.app.use('/oidclogin', async (req, res, next) => { if (req && (req as any).user) { diff --git a/OpenFlow/src/SamlProvider.ts b/OpenFlow/src/SamlProvider.ts index 6747ac5b..8b452bd5 100644 --- a/OpenFlow/src/SamlProvider.ts +++ b/OpenFlow/src/SamlProvider.ts @@ -107,7 +107,10 @@ export class SamlProvider { try { // tslint:disable-next-line: max-line-length samlp.parseRequest(req, samlpoptions, async (_err: any, samlRequestDom: any): Promise => { - res.cookie("originalUrl", req.originalUrl, { maxAge: 900000, httpOnly: true }); + try { + res.cookie("originalUrl", req.originalUrl, { maxAge: 900000, httpOnly: true }); + } catch (error) { + } res.redirect("/"); }); } catch (error) { diff --git a/OpenFlow/src/WebServer.ts b/OpenFlow/src/WebServer.ts index 50485990..1ae8e1bf 100644 --- a/OpenFlow/src/WebServer.ts +++ b/OpenFlow/src/WebServer.ts @@ -701,7 +701,7 @@ export class WebServer { } // if(res.result) res.result = Buffer.from(JSON.stringify(res.result)); // if(res.results) res.results = Buffer.from(JSON.stringify(res.results)); - if(res.result) res.result = JSON.stringify(res.result); + if(res.result && result.command != "createindex") res.result = JSON.stringify(res.result); if(res.workitem && !NoderedUtil.IsNullUndefinded(res.workitem.payload) ) { res.workitem.payload = JSON.stringify(res.workitem.payload); } diff --git a/OpenFlow/src/index.ts b/OpenFlow/src/index.ts index e76f95cf..91d09012 100644 --- a/OpenFlow/src/index.ts +++ b/OpenFlow/src/index.ts @@ -432,9 +432,9 @@ try { } catch (error) { } -let OpenAIProxy: any = null; +let OpenAPIProxy: any = null; try { - OpenAIProxy = require("./ee/OpenAIProxy"); + OpenAPIProxy = require("./ee/OpenAPIProxy"); } catch (error) { } @@ -466,8 +466,8 @@ var server: http.Server = null; if (GrafanaProxy != null) { const grafana = await GrafanaProxy.GrafanaProxy.configure(WebServer.app, span); } - if (OpenAIProxy != null) { - const OpenAI = await OpenAIProxy.OpenAIProxy.configure(WebServer.app, span); + if (OpenAPIProxy != null) { + const OpenAI = await OpenAPIProxy.OpenAPIProxy.configure(WebServer.app, span); } OAuthProvider.configure(WebServer.app, span); WebSocketServer.configure(server, span); diff --git a/OpenFlow/src/public/Chat.html b/OpenFlow/src/public/Chat.html new file mode 100644 index 00000000..f792038c --- /dev/null +++ b/OpenFlow/src/public/Chat.html @@ -0,0 +1,163 @@ + + +
+
+
+
+ +
+
+ +
+
+ +
+
+
+
+ Ask about data inside openflow. or tell the system to run openrpa workflows to connected robots. See below buttons for example conversation starters.
+ Sometimes the robot will not get the answer, then try resetting the conversation with the reset button, and try again.
+
+ +
+ + + +
+
+ +
+ + + restart + Hist + + +
+
+
+ + +
+ {{starter}} +
+
+ {{ ctrl.toolmessage.name }} / {{ ctrl.toolmessage.collectionname }} + + +
+ + + + + + + + + + + + +
{{key}}
+ + {{model[key]}} + {{model[key]}} +
diff --git a/OpenFlow/src/public/ChatThreads.html b/OpenFlow/src/public/ChatThreads.html new file mode 100644 index 00000000..0e54e912 --- /dev/null +++ b/OpenFlow/src/public/ChatThreads.html @@ -0,0 +1,35 @@ + +
+ +newchat +
+ + + + + + + + + + + + + + + + +
namecreated +
{{model.name}} + + + + +
\ No newline at end of file diff --git a/OpenFlow/src/public/CommonControllers.ts b/OpenFlow/src/public/CommonControllers.ts index 73c03d65..30461753 100644 --- a/OpenFlow/src/public/CommonControllers.ts +++ b/OpenFlow/src/public/CommonControllers.ts @@ -706,6 +706,7 @@ export class entitiesCtrl { finalor.push(newq); finalexactor.push(newexactq); } + finalexactor.push({ "_id": this.searchstring }); var hastextindex = false; if (this.WebSocketClientService.collections_with_text_index.indexOf(this.collection) > -1) { hastextindex = true; diff --git a/OpenFlow/src/public/Console.html b/OpenFlow/src/public/Console.html index a60646b0..9a6023da 100644 --- a/OpenFlow/src/public/Console.html +++ b/OpenFlow/src/public/Console.html @@ -73,7 +73,7 @@ housekeeping login_provider + ng-click="ctrl.model.log_login_provider = !ctrl.model.log_login_provider; ctrl.submit()" value="1" id="ctrl.model.log_login_provider" />LoginProvider otel { - try { - if (!this.allowclick) { - // event.cancelBubble = true; - event.stopImmediatePropagation(); - return event.preventDefault(); - } - } catch (error) { - console.error(error); + "click", + (event) => { + try { + if (!this.allowclick) { + // event.cancelBubble = true; + event.stopImmediatePropagation(); + return event.preventDefault(); } - }); + } catch (error) { + console.error(error); + } + }); + document.addEventListener('keydown', (event) => { + if(!this.PathIs(this.searchpaths)) return; + // Check if 'Ctrl' or 'Command' (for MacOS) is pressed along with 'F' + if ((event.ctrlKey || event.metaKey) && event.key === 'f') { + event.preventDefault(); // Prevent the default Ctrl+F behavior + document.getElementById('menusearch').focus(); // Focus on your search field + } + }); + this.halfmoon = require("halfmoon"); console.debug("MenuCtrl::constructor"); $scope.$root.$on('$routeChangeStart', (...args) => { this.routeChangeStart.apply(this, args); }); this.path = this.$location.path(); + this.$scope.$on('search', (event, data) => { + this.searchstring = data; + }); + this.halfmoon.onDOMContentLoaded(); const cleanup = this.$scope.$on('signin', async (event, data) => { if (event && data) { } this.user = data; this.signedin = true; + this.version = this.WebSocketClientService.version; + this.majorversion = this.version; + const dotCount = this.version.split('.').length - 1; + if(dotCount == 3){ + this.majorversion = this.version.substring(0, this.version.lastIndexOf('.')); + } + console.log(this.version) + + this.customer = this.WebSocketClientService.customer; this.customers = await NoderedUtil.Query({ collectionname: "users", query: { _type: "customer" }, orderby: { "name": 1 }, top: 20 }); @@ -228,7 +287,7 @@ export class MenuCtrl { // this.WebSocketClientService.loadToken(); this.WebSocketClientService.impersonate("-1"); } - PathIs(path: string) { + PathIs(path: string | string[]) { if (path == null && path == undefined) return false; if (this.path == null && this.path == undefined) return false; if (Array.isArray(path)) { @@ -1278,7 +1337,6 @@ export class MenuCtrl { console.error(error); } } - } export class RPAWorkflowCtrl extends entityCtrl { public arguments: any; @@ -1464,9 +1522,7 @@ export class RPAWorkflowsCtrl extends entitiesCtrl { workflow.chart = chart; if (!this.$scope.$$phase) { this.$scope.$apply(); } } - } - } download(data, filename, type) { const file = new Blob([data], { type: type }); @@ -4429,12 +4485,31 @@ export class hdrobotsCtrl extends entitiesCtrl { if (!this.$scope.$$phase) { this.$scope.$apply(); } } } -export class ClientsCtrl extends entitiesCtrl { +export class ClientsCtrl { public showinactive: boolean = false; public show: string = "all"; + public models: any[] = []; + public orderby: any = {}; + public loading: boolean = false; + public errormessage: string = ""; + public searchstring: string = ""; + public static $inject = [ + "$sce", + "$rootScope", + "$scope", + "$timeout", + "$location", + "$routeParams", + "$interval", + "WebSocketClientService", + "api", + "userdata" + ]; constructor( + public $sce: ng.ISCEService, public $rootScope: ng.IRootScopeService, public $scope: ng.IScope, + public $timeout: ng.ITimeoutService, public $location: ng.ILocationService, public $routeParams: ng.route.IRouteParamsService, public $interval: ng.IIntervalService, @@ -4442,24 +4517,12 @@ export class ClientsCtrl extends entitiesCtrl { public api: api, public userdata: userdata ) { - super($rootScope, $scope, $location, $routeParams, $interval, WebSocketClientService, api, userdata); - this.autorefresh = true; console.debug("RobotsCtrl"); - this.basequery = {}; - this.searchfields = []; - this.collection = "entities"; - this.pagesize = 1; - this.postloadData = this.processdata; - if (this.userdata.data.ClientsCtrl) { - this.basequery = this.userdata.data.ClientsCtrl.basequery; - this.collection = this.userdata.data.ClientsCtrl.collection; - this.baseprojection = this.userdata.data.ClientsCtrl.baseprojection; - this.orderby = this.userdata.data.ClientsCtrl.orderby; - this.searchstring = this.userdata.data.ClientsCtrl.searchstring; - this.basequeryas = this.userdata.data.ClientsCtrl.basequeryas; - this.showinactive = this.userdata.data.ClientsCtrl.showinactive; - this.show = this.userdata.data.ClientsCtrl.show; - } + this.$scope.$on('search', (event, data) => { + this.searchstring = data; + this.processdata(); + }); + WebSocketClientService.onSignedin((user: TokenUser) => { // this.loadData(); this.processdata() @@ -4469,15 +4532,17 @@ export class ClientsCtrl extends entitiesCtrl { var result = await NoderedUtil.CustomCommand({ "command": "getclients" }); this.models = result as any; if (!this.userdata.data.ClientsCtrl) this.userdata.data.ClientsCtrl = {}; - this.userdata.data.ClientsCtrl.basequery = this.basequery; - this.userdata.data.ClientsCtrl.collection = this.collection; - this.userdata.data.ClientsCtrl.baseprojection = this.baseprojection; - this.userdata.data.ClientsCtrl.orderby = this.orderby; - this.userdata.data.ClientsCtrl.searchstring = this.searchstring; - this.userdata.data.ClientsCtrl.basequeryas = this.basequeryas; this.userdata.data.ClientsCtrl.showinactive = this.showinactive; this.userdata.data.ClientsCtrl.show = this.show; + if(this.searchstring != "") { + this.models = this.models.filter(x => + x.name.toLowerCase().indexOf(this.searchstring.toLowerCase()) > -1 + || x.username.toLowerCase().indexOf(this.searchstring.toLowerCase()) > -1 + || x.user?.email?.toLowerCase().indexOf(this.searchstring.toLowerCase()) > -1 + ); + } + if (this.orderby != null) { var keys = Object.keys(this.orderby); @@ -4522,7 +4587,6 @@ export class ClientsCtrl extends entitiesCtrl { try { this.loading = true; await this.WebSocketClientService.impersonate(model._id); - this.loadData(); } catch (error) { this.errormessage = JSON.stringify(error); } @@ -7348,6 +7412,7 @@ export class ConfigCtrl extends entityCtrl { {"name": "enable_openai", "type": "boolean", "default": "false"}, {"name": "enable_openapi", "type": "boolean", "default": "true"}, {"name": "enable_openapiauth", "type": "boolean", "default": "true"}, + {"name": "llmchat_queue", "type": "string", "default": ""}, {"name": "openai_token", "type": "string", "default": ""}, {"name": "log_with_colors", "type": "boolean", "default": "true"}, {"name": "log_database_queries_to_collection", "type": "string", "default": ""}, @@ -8844,4 +8909,786 @@ export class RunPackageCtrl extends entityCtrl { var _a = this.agents.find(x => x._id == this.id); await NoderedUtil.Queue({ data: {"command": "addcommandstreamid"}, queuename: _a.slug + "agent" }); } +} + +export class QueryCtrl { + public queuename: string = ""; + public pipeline: string = ""; + public model: any = null; + public models: any[] = []; + public collections: any[] = []; + public keys: string[] = []; + public collection: string = ""; + public searchstring: string = ""; + public reasoning: string = ""; + public errormessage: string = ""; + public errorcount: number = 0; + public loadingollama: boolean = false; + public loadingdata: boolean = false; + + public static $inject = [ + "$rootScope", + "$scope", + "$location", + "$routeParams", + "$interval", + "WebSocketClientService", + "api", + "userdata" + ]; + constructor( + public $rootScope: ng.IRootScopeService, + public $scope: ng.IScope, + public $location: ng.ILocationService, + public $routeParams: ng.route.IRouteParamsService, + public $interval: ng.IIntervalService, + public WebSocketClientService: WebSocketClientService, + public api: api, + public userdata: userdata + ) { + console.debug("QueryCtrl"); + console.log(WebSocketClientService.llmchat_queue); + this.collection = $routeParams.collection; + if(this.collection == null || this.collection == "") { + this.collection = "entities"; + } + this.$scope.$on('search', (event, data) => { + this.searchstring = data; + }); + WebSocketClientService.onSignedin(async (user: TokenUser) => { + this.collections = await NoderedUtil.ListCollections({}); + if (!this.$scope.$$phase) { this.$scope.$apply(); } + await this.RegisterQueue(); + this.$scope.$on('signin', async (event, data) => { + this.collections = await NoderedUtil.ListCollections({}); + if (!this.$scope.$$phase) { this.$scope.$apply(); } + this.RegisterQueue(); + if (!this.$scope.$$phase) { this.$scope.$apply(); } + }); + if (!this.$scope.$$phase) { this.$scope.$apply(); } + }); + } + SelectCollection() { + // if (!this.userdata.data.EntitiesCtrl) this.userdata.data.EntitiesCtrl = {}; + // this.userdata.data.EntitiesCtrl.collection = this.collection; + this.$location.path("/Query/" + this.collection); + this.searchstring = ""; + this.errormessage = ""; + this.pipeline = ""; + if (!this.$scope.$$phase) { this.$scope.$apply(); } + // this.loadData(); + } + private lastsearch = ""; + private messages: chatmessage[] = []; + async Search() { + this.$rootScope.$broadcast("search", this.searchstring); + if(this.searchstring == null || this.searchstring.trim() == "") return; + if(this.lastsearch != this.searchstring) { + this.errormessage = ""; + this.pipeline = ""; + this.lastsearch = this.searchstring + } + if(this.errorcount > 3) { + this.errormessage = "Too many errors, please rephrase your question\n" + this.errormessage; + this.errorcount = 0; + return; + } + if(this.errormessage.includes("Too many errors")) { + this.errormessage = ""; + this.pipeline = ""; + } + this.loadingollama = true; + if(this.errormessage != "") { + } else { + this.errormessage = ""; + this.pipeline = ""; + } + this.models = []; + this.keys = []; + if (!this.$scope.$$phase) { this.$scope.$apply(); } + + let prompt = ""; + const lastweek = new Date(); + prompt = "Generate mongodb aggregation pipeline, based on the user input, reply with the generated pipeline in json format using syntax `{\"pipeline\": [${pipeline}], \"reasoning\": \"explain why you choose this query\": }`\n" + + `Today is ${new Date().toISOString()}\n`; + + prompt += + `## Formatting Standards for ASCII Tables + - Standardize column widths and text alignment. + - Include headers for columns. + - Handle long text through truncation or wrapping, as appropriate. + ## Additional Notes + - Use case-insensitive searches for matching operations. + - We are creating a query toward a the '${this.collection}' collection` + if (this.WebSocketClientService.timeseries_collections.indexOf(this.collection) != -1) { + prompt += ` - '${this.collection}' is a time series collection and has a lot of data, so always use '$group' if there is a chance of returning many results\n` + } + prompt += `## object schema + All objects in the collection have a _type and name property, for instance in openrpa collection, we have "_type": "workflow", "_type": "project" and in users collection we have "_type": "user" "_type": "role" etc. + - '_created': date field for when created + - '_createdby': string, with name of user who created this object + - '_createdbyid': string with the _id of the user who created this object + - '_modified': date field for when modified + - '_modifiedby': string, with name of user who modified this object + - '_modifiedbyid': string with the _id of the user who modified this object + - '_type': string, with the type of object, for instance 'user', 'role', 'workflow', 'project' etc. + - 'name': string, with the name of the object` + // if (this.WebSocketClientService.timeseries_collections.indexOf(this.collection) != -1) { + // // } else if(this.WebSocketClientService.collections_with_text_index.indexOf(this.collection) != -1) { + // prompt += `We are querying the time series collection '${this.collection}' that has a lot of data, so always use aggregates if there is a chance of returning many results\n` + // } else { + // prompt += `We are querying the ''${this.collection}' collection. All objects in the collection have a _type and name property, for instance in openrpa collection, we have "_type": "workflow", "_type": "project" and in users collection we have "_type": "user" "_type": "role" etc.\n` + // } + // prompt += `All objects have a datatime filed '_created' and '_modified' so decided if you need to use one of those fields. If user asks for all new users created the last week we use "_created": {"$gt": "${lastweek.toISOString()}"} \n` + + // "Carefully decided if we should use a group by or not\n" + + // "if user asks you to use type or _type, user most likely mean the field `_type`\n" + + // "if user asks for created, use field `_created`\n" + + // "if user asks for modified or updated, use field `_modified`\n" + + // "IMPORTANT! Never use $loopup. Never remove `_` from field names! \n" + // "IMPORTANT! Never use $loopup. Never remove _ if field has _ in it\n" + prompt += "### User prompt: " + this.searchstring; + if(this.errormessage != "") { + console.log("Adding last error and pipeline to prompt") + this.errorcount++; + prompt += "### Last Query: " + this.pipeline + prompt += "### Last Error: " + this.errormessage + // this.errormessage = ""; + this.pipeline = ""; + if (!this.$scope.$$phase) { this.$scope.$apply(); } + + } else { + } + + lastweek.setDate(lastweek.getDate() - 7); + var payload = { + func: "generate", + model: "mistral", + prompt, + raw: false, + json: true, + } + try { + const result: any = await NoderedUtil.Queue({ + queuename: this.WebSocketClientService.llmchat_queue, replyto: this.queuename, + data: payload, + striptoken: true }); + } catch (error) { + this.errormessage = error.message ? error.message : error; + } + } + async RunQuery() { + try { + let pipeline = null; + try { + pipeline = JSON.parse(this.pipeline); + } catch (error) { + this.errormessage = error.message ? error.message : error; + return; + } + if(pipeline == null) { + console.log("pipeline is null") + return; + } + this.loadingdata = true; + this.models = []; + this.keys = []; + this.errormessage = ""; + if (!this.$scope.$$phase) { this.$scope.$apply(); } + + this.models = await NoderedUtil.Aggregate({ collectionname: this.collection, + aggregates: pipeline + }); + if(this.models.length > 0) { + var __keys = Object.keys(this.models[0]); + var _keys = __keys.filter(x => x != "name" && x.startsWith("_") == false); + + if(__keys.includes("_created")) { + _keys.unshift("_created"); + } + if(__keys.includes("_type")) { + _keys.unshift("_type"); + } + if(__keys.includes("name")) { + _keys.unshift("name"); + } + if(__keys.includes("_id")) { + _keys.unshift("_id"); + } + if(_keys.length > 7) { + _keys = _keys.slice(0, 7); + } + this.keys = _keys; + } + if(this.models.length == 0) { + this.models = [{name: "No results"}] + this.keys = ["name"] + } + this.errorcount = 0; + this.loadingdata = false + } catch (error) { + this.errormessage = error.message ? error.message : error; + } + this.loadingdata = false + if (!this.$scope.$$phase) { this.$scope.$apply(); } + } + async RegisterQueue() { + if(this.queuename != "") return; + this.queuename = await NoderedUtil.RegisterQueue({ + callback: async (_data: QueueMessage, ack: any) => { + ack(); + try { + if(_data == null) return; + var correlationId = _data.correlationId; + var data: any = _data; + if(data.data != null) data = data.data; + if(data.error != null && data.error != "") { + this.errormessage = data.error; + this.loadingollama = false; + return; + } + if(data.func == "generating") { + // console.log(data); + this.pipeline += data.response; + } else if(data.func == "generate") { + if(data.response == null || data.response == "") { + this.errormessage = "No response from LLM"; + this.loadingollama = false; + return; + } + try { + var pipeline = JSON.parse(data.response); + this.pipeline =JSON.stringify(pipeline.pipeline, null, 2); + this.reasoning = pipeline.reasoning; + this.loadingollama = false + this.RunQuery() + } catch (error) { + this.errormessage = error.message ? error.message : error; + this.loadingollama = false; + this.errorcount =0; + console.log(data.response); + console.error(error); + } + } else { + console.log(data); + } + } catch (error) { + this.errormessage = error.message ? error.message : error; + this.loadingollama = false; + } finally { + if (!this.$scope.$$phase) { this.$scope.$apply(); } + } + }, closedcallback: (msg) => { + this.queuename = ""; + setTimeout(this.RegisterQueue.bind(this), (Math.floor(Math.random() * 6) + 1) * 500); + } + }); + console.log("RegisterQueue", this.queuename); + } + OpenEntity(model) { + this.$location.path("/Entity/" + this.collection + "/" + model._id); + if (!this.$scope.$$phase) { this.$scope.$apply(); } + return; + } + +} + + + +export class ChatCtrl { + public queuename: string = ""; + public llmmodel: string = "openai/gpt-3.5-turbo-1106"; + public model: any = null; + public models: any[] = []; + public collections: any[] = []; + public keys: string[] = []; + public chatmessage: string = ""; + public errormessage: string = ""; + public errorcount: number = 0; + public loadingollama: boolean = false; + public loadingdata: boolean = false; + public starters: string[] = []; + + public static $inject = [ + "$sce", + "$rootScope", + "$scope", + "$timeout", + "$location", + "$routeParams", + "$interval", + "WebSocketClientService", + "api", + "userdata" + ]; + constructor( + public $sce: ng.ISCEService, + public $rootScope: ng.IRootScopeService, + public $scope: ng.IScope, + public $timeout: ng.ITimeoutService, + public $location: ng.ILocationService, + public $routeParams: ng.route.IRouteParamsService, + public $interval: ng.IIntervalService, + public WebSocketClientService: WebSocketClientService, + public api: api, + public userdata: userdata + ) { + console.debug("QueryCtrl"); + console.log(WebSocketClientService.llmchat_queue); + + this.threadid = $routeParams.threadid; + if(this.threadid == null) this.threadid = ""; + + this.starters = [ + "Find the email of user named macuser", + "What are the last 20 audit entries ?", + "list the number of audit entries, grouped by month", + "get then top 20 OpenRPA workflows grouped by created user", + "What is the top 10 most run openrpa workflow grouped by name?", + "What is the top 10 most run openrpa workflow grouped by name, and then write a short story about OpenRPA the happy robot" + ] + var _llmmodel = this.getCookie("llmchatmodel"); + if(_llmmodel != null && _llmmodel != "") { + this.llmmodel = _llmmodel; + } + // this.collection = $routeParams.collection; + WebSocketClientService.onSignedin(async (user: TokenUser) => { + var workflows = await NoderedUtil.Query({ collectionname: "openrpa", query: { _type: "workflow" }, top:1 }); + var workflow = "WhoAmI"; + var robotuser = user.username; + if(workflows.length > 0) { + workflow = workflows[0].name; + } + console.log("signed in", user) + if(user.username == "az") { + workflow = "WhoAmI"; + robotuser = "macuser" + } + this.starters = [ + "Find the email of user named " + user.username, + "what is the most run openrpa workflow ?", + "Run `" + workflow + "` on `" + robotuser + "`", + "What are the last 20 audit entries ?", + "list the number of audit entries, grouped by month", + "get then number of OpenRPA workflows grouped by created user", + "What is the top 10 most run openrpa workflow grouped by name?", + "What is the top 20 most failed openrpa workflow grouped by name?", + "What is the top 10 most run openrpa workflow grouped by name, and then write a short story about OpenRPA the happy robot" + ] + this.LoadThread(); + this.collections = await NoderedUtil.ListCollections({}); + if (!this.$scope.$$phase) { this.$scope.$apply(); } + await this.RegisterQueue(); + this.$scope.$on('signin', async (event, data) => { + this.collections = await NoderedUtil.ListCollections({}); + this.LoadThread(); + if (!this.$scope.$$phase) { this.$scope.$apply(); } + this.RegisterQueue(); + if (!this.$scope.$$phase) { this.$scope.$apply(); } + }); + if (!this.$scope.$$phase) { this.$scope.$apply(); } + this.$timeout(()=> { + var input = document.getElementById('chatmessage'); + input.focus(); + }, 200) + }); + // Watch for changes in your messages array + $scope.$watchCollection('ctrl.messages', (newMessages: any, oldMessages: any) => { + if (newMessages.length !== oldMessages.length) { + $timeout(this.scrollToBottom, 100); // Scroll after the DOM update + } + }); + } + async LoadThread() { + if(this.threadid != "") { + var _messages = await NoderedUtil.Query({ collectionname: "llmchat", query: { threadid: this.threadid, "_type": "message" }, top:100 }); + _messages.sort((a, b) => { + return a.message.index - b.message.index; + }); + this.messages = _messages.map((x) => x.message);; + if (!this.$scope.$$phase) { this.$scope.$apply(); } + } + } + Reset() { + this.errormessage = ""; + this.errorcount = 0; + this.chatmessage = ""; + this.threadid = ""; + this.toolmessage = null; + this.messages = []; + this.models = []; + this.keys = []; + if (!this.$scope.$$phase) { this.$scope.$apply(); } + } + scrollToBottom() { + var chatContainer = document.querySelector('.chat-container'); + chatContainer.scrollTop = chatContainer.scrollHeight; + } + setCookie(cname, cvalue, exdays) { + const d = new Date(); + d.setTime(d.getTime() + (exdays * 24 * 60 * 60 * 1000)); + const expires = "expires=" + d.toUTCString(); + document.cookie = cname + "=" + cvalue + ";" + expires + ";path=/"; + } + getCookie(cname) { + const name = cname + "="; + const decodedCookie = decodeURIComponent(document.cookie); + const ca = decodedCookie.split(';'); + for (let i = 0; i < ca.length; i++) { + let c = ca[i]; + while (c.charAt(0) == ' ') { + c = c.substring(1); + } + if (c.indexOf(name) == 0) { + return c.substring(name.length, c.length); + } + } + return ""; + } + private collectionname: string = ""; + private toolmessage:chatmessage = null; + async runtool(message: any) { + try { + this.loadingdata = true; + this.models = []; + this.keys = []; + this.toolmessage = message; + if (!this.$scope.$$phase) { this.$scope.$apply(); } + + if(message.name == "MongoAggregate") { + var pipeline = message.pipeline; + var collectionname = message.collectionname; + this.collectionname = collectionname; + if(pipeline != null && collectionname != null) { + this.models = await NoderedUtil.Aggregate({ collectionname: message.collectionname, + aggregates: pipeline + }); + } else { + this.errormessage = message.content; + } + } else if(message.name == "MongoQuery") { + var query = message.query; + var top = message.top; + var collectionname = message.collectionname; + this.collectionname = collectionname; + var projection = message.projection; + if(query == null) query = {}; + console.log(collectionname, query, top, projection); + if(query != null && collectionname != null) { + this.models = await NoderedUtil.Query({ collectionname, query, top, projection }); + } else { + this.errormessage = message.content; + } + } else if(message.name == "GetCollections") { + this.models = await NoderedUtil.ListCollections({}); + } else if(message.name == "RunOpenRPAWorkflow") { + console.log("runtool RunOpenRPAWorkflow", message); + if(message.correlationId == null || message.correlationId == "") { + message.correlationId = NoderedUtil.GetUniqueIdentifier(); + } + var div = document.getElementById(message.correlationId); + if(div != null) { + div.innerText = "Sending invoke command to " + message.robotid + "\n"; + } else { + this.$timeout(()=> { + var div = document.getElementById(message.correlationId); + if(div != null) { + div.innerText = "Sending invoke command to " + message.robotid + "\n"; + } + }, 200); + } + + const rpacommand = { + command: "invoke", + workflowid: message.workflowid, + data: message.parameters + } + await NoderedUtil.Queue({ + correlationId: message.correlationId, + data: rpacommand, + queuename: message.robotid, + replyto: this.queuename, + }) + } + if(this.models.length > 0) { + var __keys = Object.keys(this.models[0]); + var _keys = __keys.filter(x => x != "name" && x.startsWith("_") == false); + + if(__keys.includes("_created")) { + _keys.unshift("_created"); + } + if(__keys.includes("_type")) { + _keys.unshift("_type"); + } + if(__keys.includes("name")) { + _keys.unshift("name"); + } + if(__keys.includes("_id")) { + _keys.unshift("_id"); + } + if(_keys.length > 7) { + _keys = _keys.slice(0, 7); + } + this.keys = _keys; + } + if(this.models.length == 0) { + this.models = [{name: "No results"}] + this.keys = ["name"] + } + this.errorcount = 0; + this.loadingdata = false + } catch (error) { + this.errormessage = error.message ? error.message : error; + } + this.$timeout(()=> { + var table = document.getElementById('table1'); // Adjust the ID if necessary + table.scrollTop = table.scrollHeight; + }, 200) + + this.loadingdata = false + if (!this.$scope.$$phase) { this.$scope.$apply(); } + } + + Markdown(text) { + // @ts-ignore + var converter = new showdown.Converter(), + html = converter.makeHtml(text) + if(html != null) { + html = html.split("\n").join("
"); + } + return this.$sce.trustAsHtml(html); + } + + async Resubmit(message) { + var index = this.messages.findIndex(x => x == message); + if(index == -1) return; + // delete this message and everythig after + this.messages = this.messages.slice(0, index); + this.chatmessage = message.content; + this.Search(); + } + + private lastmessage = ""; + private messages: chatmessage[] = []; + private threadid: string = ""; + async Search() { + if(this.chatmessage == null || this.chatmessage.trim() == "") return; + if(this.lastmessage != this.chatmessage) { + this.errormessage = ""; + this.lastmessage = this.chatmessage + } + if(this.errorcount > 3) { + this.errormessage = "Too many errors, please rephrase your question\n" + this.errormessage; + this.errorcount = 0; + return; + } + // if new question was sent, clear error + if(this.errormessage.includes("Too many errors")) { + this.errormessage = ""; + } + this.loadingollama = true; + if(this.errormessage != "") { + } else { + this.errormessage = ""; + } + this.models = []; + this.keys = []; + if (!this.$scope.$$phase) { this.$scope.$apply(); } + + this.setCookie("llmchatmodel", this.llmmodel, 365); + + var payload = { + func: "chat", + model: this.llmmodel, + // model: "ollama/mistral", + // model: "ollama/functionary", + message: this.chatmessage, + threadid: this.threadid, + // json: true, + } + try { + const result: any = await NoderedUtil.Queue({ + queuename: this.WebSocketClientService.llmchat_queue, replyto: this.queuename, + data: payload }); + } catch (error) { + this.loadingollama = false; + this.errormessage = error.message ? error.message : error; + console.error(error); + } + if (!this.$scope.$$phase) { this.$scope.$apply(); } + } + lasttoolindex = 0; + async checkForNewTools() { + this.$timeout(()=> { + var input = document.getElementById('chatmessage'); + input.focus(); + }, 200) + // if one of the last 4 messages is from role "tool" then + // for(let y = this.messages.length - 1; y > this.lasttoolindex; y--) { + // console.log(y); + // var toolmessage = this.messages[y]; + // if(toolmessage.role == "tool") { + // this.lasttoolindex = y; + // this.runtool(toolmessage); + // break; + // } + // } + } + async RegisterQueue() { + if(this.queuename != "") return; + try { + this.queuename = await NoderedUtil.RegisterQueue({ + callback: async (_data: QueueMessage, ack: any) => { + ack(); + try { + if(_data == null) return; + var correlationId = _data.correlationId; + var data: any = _data; + if(data.data != null) data = data.data; + if(data.threadid != null && data.threadid != "") { + this.threadid = data.threadid; + // this.$location.path("/Chat/" + this.threadid); + } + if(data.error != null && data.error != "") { + console.log("ERROR") + console.error(data.error); + this.errormessage = data.error; + this.loadingollama = false; + if(data.messages != null) { + this.messages = data.messages; + } + this.checkForNewTools() + return; + } + if(data.func == "chat") { + console.log(data); + if(data.messages != null) { + this.messages = data.messages; + console.log(this.messages); + } else if(data.message != null) { + console.log(data.message); + // @ts-ignore + this.messages = this.messages.filter(x => x.temp != true); + this.messages.push(data.message); + if(data.message.role == "tool") { + if(data.message.name != "RunOpenRPAWorkflow") { + this.runtool(data.message); + } + } + } + this.chatmessage = ""; + this.loadingollama = false; + this.errormessage = ""; + this.checkForNewTools() + return; + } else if(data.func == "message") { + if(data.message != null) { + // @ts-ignore + this.messages = this.messages.filter(x => x.temp != true); + this.messages.push(data.message); + if(data.message.role == "tool") { + if(data.message.name != "RunOpenRPAWorkflow") { + this.runtool(data.message); + } + } + this.errormessage = ""; + } + } else if(data.func == "messages") { + if(data.messages != null) { + this.messages = data.messages; + } + console.log(this.messages); + this.errormessage = ""; + this.checkForNewTools() + } else if(data.func == "generating") { + // @ts-ignore + var temp = this.messages.find(x => x.temp == true); + if(temp == null) { + // @ts-ignore + temp = {"role": "assistant", "content": "", temp: true}; + this.messages.push(temp); + } + temp.content += data.response; + this.$timeout(this.scrollToBottom, 100); + } else { + if(correlationId != null && correlationId != "") { + var div = document.getElementById(correlationId); + if(div != null) { + let m = ""; + if(data.command == "invokesuccess" || data.command == "invokecompleted" ) { + try { + m = JSON.stringify(data.data); + } catch (error) { + } + } else if(data.command == "timeout") { + m = "TIMEOUT! is robot running ?" + } + div.innerText += data.command + ": " + m + "\n"; + } + } + console.log(data); + } + } catch (error) { + this.errormessage = error.message ? error.message : error; + this.loadingollama = false; + } finally { + if (!this.$scope.$$phase) { this.$scope.$apply(); } + } + }, closedcallback: (msg) => { + this.queuename = ""; + setTimeout(this.RegisterQueue.bind(this), (Math.floor(Math.random() * 6) + 1) * 500); + } + }); + } catch (error) { + this.errormessage = error.message ? error.message : error; + console.error(error); + } + console.log("RegisterQueue", this.queuename); + } + OpenEntity(model) { + this.$location.path("/Entity/" + this.collectionname + "/" + model._id); + if (!this.$scope.$$phase) { this.$scope.$apply(); } + return; + } + +} +export class ChatThreadsCtrl extends entitiesCtrl { + constructor( + public $rootScope: ng.IRootScopeService, + public $scope: ng.IScope, + public $location: ng.ILocationService, + public $routeParams: ng.route.IRouteParamsService, + public $interval: ng.IIntervalService, + public WebSocketClientService: WebSocketClientService, + public api, + public userdata: userdata + ) { + super($rootScope, $scope, $location, $routeParams, $interval, WebSocketClientService, api, userdata); + console.debug("ChatThreadsCtrl"); + this.basequery = { _type: "thread" }; + this.collection = "llmchat"; + this.baseprojection = { _type: 1, type: 1, name: 1, _created: 1, _createdby: 1, _modified: 1, dbusage: 1 }; + this.postloadData = this.processData; + if (this.userdata.data.ChatThreadsCtrl) { + this.basequery = this.userdata.data.ChatThreadsCtrl.basequery; + this.collection = this.userdata.data.ChatThreadsCtrl.collection; + this.baseprojection = this.userdata.data.ChatThreadsCtrl.baseprojection; + this.orderby = this.userdata.data.ChatThreadsCtrl.orderby; + this.searchstring = this.userdata.data.ChatThreadsCtrl.searchstring; + this.basequeryas = this.userdata.data.ChatThreadsCtrl.basequeryas; + } + WebSocketClientService.onSignedin((user: TokenUser) => { + this.loadData(); + }); + } + async processData(): Promise { + if (!this.userdata.data.ChatThreadsCtrl) this.userdata.data.ChatThreadsCtrl = {}; + this.userdata.data.ChatThreadsCtrl.basequery = this.basequery; + this.userdata.data.ChatThreadsCtrl.collection = this.collection; + this.userdata.data.ChatThreadsCtrl.baseprojection = this.baseprojection; + this.userdata.data.ChatThreadsCtrl.orderby = this.orderby; + this.userdata.data.ChatThreadsCtrl.searchstring = this.searchstring; + this.userdata.data.ChatThreadsCtrl.basequeryas = this.basequeryas; + this.loading = false; + if (!this.$scope.$$phase) { this.$scope.$apply(); } + } } \ No newline at end of file diff --git a/OpenFlow/src/public/Customer.html b/OpenFlow/src/public/Customer.html index e89150a0..1eb5650e 100644 --- a/OpenFlow/src/public/Customer.html +++ b/OpenFlow/src/public/Customer.html @@ -49,6 +49,13 @@

customer

{{ctrl.model.vatnumber}} +
+
+
+ +
+
Forced domains
diff --git a/OpenFlow/src/public/Entities.html b/OpenFlow/src/public/Entities.html index 0746419b..d632d297 100644 --- a/OpenFlow/src/public/Entities.html +++ b/OpenFlow/src/public/Entities.html @@ -28,6 +28,10 @@ + + diff --git a/OpenFlow/src/public/Query.html b/OpenFlow/src/public/Query.html new file mode 100644 index 00000000..a58ec966 --- /dev/null +++ b/OpenFlow/src/public/Query.html @@ -0,0 +1,48 @@ + + +
+
+ + +
+
+ +
+ + {{ctrl.searchstring}} + + +
+
+
+
+ + + +
+
{{ctrl.pipeline}}
+
{{ctrl.reasoning}}
+ + + + + + + + + + + + +
{{key}}
+ + {{model[key]}} + {{model[key]}} +
diff --git a/OpenFlow/src/public/WebSocketClientService.ts b/OpenFlow/src/public/WebSocketClientService.ts index cb77ebc7..cfc76588 100644 --- a/OpenFlow/src/public/WebSocketClientService.ts +++ b/OpenFlow/src/public/WebSocketClientService.ts @@ -50,6 +50,7 @@ export class WebSocketClientService { this.ping_clients_interval = data.ping_clients_interval; this.validlicense = data.validlicense; this.grafana_url = data.grafana_url + this.llmchat_queue = data.llmchat_queue if (NoderedUtil.IsNullUndefinded(WebSocketClient.instance)) { @@ -92,6 +93,21 @@ export class WebSocketClientService { } try { const result = await NoderedUtil.SigninWithToken({ jwt: data.jwt, rawAssertion: data.rawAssertion }); + // @ts-ignore + if(result == null || (result.message && result.message.includes("not validated")) || result.user == null) { + this.setCookie("validateurl", this.$location.path(), 365); + setTimeout(() => { + top.location.href = '/login'; + document.write('') + }, 500); + try { + document.write(error); + document.write("
Signout"); + } catch (error) { + + } + return; + } this.customer = null; if (!NoderedUtil.IsNullUndefinded(WebSocketClient.instance.user) && !NoderedUtil.IsNullEmpty(WebSocketClient.instance.user.selectedcustomerid)) { @@ -196,6 +212,7 @@ export class WebSocketClientService { public ping_clients_interval: number = 10000; public validlicense: boolean = false; public grafana_url: string = ""; + public llmchat_queue: string = ""; getJSON(url: string, callback: any): void { const xhr: XMLHttpRequest = new XMLHttpRequest(); diff --git a/OpenFlow/src/public/Workitem.html b/OpenFlow/src/public/Workitem.html index 9874e499..1631ee30 100644 --- a/OpenFlow/src/public/Workitem.html +++ b/OpenFlow/src/public/Workitem.html @@ -63,7 +63,6 @@

{{ctrl.model.name}}

- diff --git a/OpenFlow/src/public/app.ts b/OpenFlow/src/public/app.ts index 569a07ac..1fe3e58a 100644 --- a/OpenFlow/src/public/app.ts +++ b/OpenFlow/src/public/app.ts @@ -7,7 +7,7 @@ import { hdrobotsCtrl, ClientsCtrl, AuditlogsCtrl, CredentialsCtrl, CredentialCtrl, DuplicatesCtrl, OAuthClientsCtrl, OAuthClientCtrl, DeletedCtrl, CustomerCtrl, EntityRestrictionsCtrl, EntityRestrictionCtrl, CustomersCtrl, ResourcesCtrl, ResourceCtrl, WorkitemsCtrl, WorkitemCtrl, WorkitemQueuesCtrl, WorkitemQueueCtrl, MailHistCtrl, MailHistsCtrl, FormResourcesCtrl, FormResourceCtrl, - WebsocketClientsCtrl, ConsoleCtrl, AgentsCtrl, AgentCtrl, PackagesCtrl, PackageCtrl, RunPackageCtrl, ConfigCtrl + WebsocketClientsCtrl, ConsoleCtrl, AgentsCtrl, AgentCtrl, PackagesCtrl, PackageCtrl, RunPackageCtrl, ConfigCtrl, QueryCtrl, ChatCtrl, ChatThreadsCtrl } from "./Controllers"; require('angular-route'); @@ -100,6 +100,15 @@ module openflow { .when('/Entities/:collection', { templateUrl: 'Entities.html', controller: EntitiesCtrl, controllerAs: 'ctrl' }) .when('/Entity/:collection', { templateUrl: 'Entity.html', controller: EntityCtrl, controllerAs: 'ctrl' }) .when('/Entity/:collection/:id', { templateUrl: 'Entity.html', controller: EntityCtrl, controllerAs: 'ctrl' }) + .when('/Query', { templateUrl: 'Query.html', controller: QueryCtrl, controllerAs: 'ctrl' }) + .when('/Query/:collection', { templateUrl: 'Query.html', controller: QueryCtrl, controllerAs: 'ctrl' }) + + .when('/ChatThreads', { templateUrl: 'ChatThreads.html', controller: ChatThreadsCtrl, controllerAs: 'ctrl' }) + .when('/Chat', { templateUrl: 'Chat.html', controller: ChatCtrl, controllerAs: 'ctrl' }) + .when('/Chat/:threadid', { templateUrl: 'Chat.html', controller: ChatCtrl, controllerAs: 'ctrl' }) + + + .when('/Duplicates/:collection', { templateUrl: 'Duplicates.html', controller: DuplicatesCtrl, controllerAs: 'ctrl' }) diff --git a/OpenFlow/src/public/index.html b/OpenFlow/src/public/index.html index dc5f0ff3..d933ecab 100644 --- a/OpenFlow/src/public/index.html +++ b/OpenFlow/src/public/index.html @@ -16,17 +16,22 @@ data-dm-shortcut-enabled="true" data-set-preferred-mode-onload="true">
-
+ + + chat agents + + Misc management sitename - v{{menuctrl.WebSocketClientService.version}} + v{{menuctrl.majorversion}} + + +
diff --git a/OpenFlow/src/public/languages/da-DK/web.json b/OpenFlow/src/public/languages/da-DK/web.json index ce52dc80..30362e43 100644 --- a/OpenFlow/src/public/languages/da-DK/web.json +++ b/OpenFlow/src/public/languages/da-DK/web.json @@ -156,5 +156,7 @@ "formvalidated": "Form valideret", "formresources": "Form ressourcer", "formresource": "Form ressource", - "addformresource": "Tilføj form ressource" + "addformresource": "Tilføj form ressource", + "chat": "Chat", + "newchat": "Ny chat" } \ No newline at end of file diff --git a/OpenFlow/src/public/languages/en-US/web.json b/OpenFlow/src/public/languages/en-US/web.json index 2933bfc6..ad18817a 100644 --- a/OpenFlow/src/public/languages/en-US/web.json +++ b/OpenFlow/src/public/languages/en-US/web.json @@ -155,5 +155,7 @@ "formvalidated": "Form validated", "formresources": "Form resources", "formresource": "Form resource", - "addformresource": "Add form resource" + "addformresource": "Add form resource", + "chat": "Chat", + "newchat": "New chat" } \ No newline at end of file diff --git a/OpenFlow/src/public/swagger.json b/OpenFlow/src/public/swagger.json new file mode 100644 index 00000000..46318536 --- /dev/null +++ b/OpenFlow/src/public/swagger.json @@ -0,0 +1,1192 @@ +{ + "components": { + "examples": {}, + "headers": {}, + "parameters": {}, + "requestBodies": {}, + "responses": {}, + "schemas": { + "Ace": { + "properties": { + "deny": { + "type": "boolean" + }, + "_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "rights": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number", + "format": "double" + } + ] + } + }, + "required": [ + "deny", + "_id", + "name", + "rights" + ], + "type": "object", + "additionalProperties": false + }, + "User": { + "properties": { + "_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "_created": { + "type": "string", + "format": "date-time" + }, + "_createdby": { + "type": "string" + }, + "_createdbyid": { + "type": "string" + }, + "_modified": { + "type": "string", + "format": "date-time" + }, + "_modifiedby": { + "type": "string" + }, + "_modifiedbyid": { + "type": "string" + }, + "_type": { + "type": "string" + }, + "_acl": { + "items": { + "$ref": "#/components/schemas/Ace" + }, + "type": "array" + }, + "_name": { + "type": "string" + }, + "_encrypt": { + "items": { + "type": "string" + }, + "type": "array" + }, + "_version": { + "type": "number", + "format": "double" + }, + "email": { + "type": "string" + }, + "username": { + "type": "string" + }, + "dbusage": { + "type": "number", + "format": "double" + }, + "dblocked": { + "type": "boolean" + }, + "newpassword": { + "type": "string" + }, + "federationids": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "_id", + "name", + "_created", + "_createdby", + "_createdbyid", + "_modified", + "_modifiedby", + "_modifiedbyid", + "_type", + "_acl", + "_name", + "_encrypt", + "_version", + "email", + "username", + "dbusage", + "dblocked", + "newpassword", + "federationids" + ], + "type": "object", + "additionalProperties": false + }, + "Pick_User.name-or-email-or-username-or-newpassword_": { + "properties": { + "name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "username": { + "type": "string" + }, + "newpassword": { + "type": "string" + } + }, + "required": [ + "name", + "email", + "username", + "newpassword" + ], + "type": "object", + "description": "From T, pick a set of properties whose keys are in the union K" + }, + "UserCreationParams": { + "$ref": "#/components/schemas/Pick_User.name-or-email-or-username-or-newpassword_" + }, + "WorkflowParameter": { + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string" + }, + "direction": { + "type": "string", + "enum": [ + "in", + "out", + "inout" + ] + } + }, + "required": [ + "name", + "type", + "direction" + ], + "type": "object", + "additionalProperties": false + }, + "Workflow": { + "properties": { + "_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "_created": { + "type": "string", + "format": "date-time" + }, + "_createdby": { + "type": "string" + }, + "_createdbyid": { + "type": "string" + }, + "_modified": { + "type": "string", + "format": "date-time" + }, + "_modifiedby": { + "type": "string" + }, + "_modifiedbyid": { + "type": "string" + }, + "_type": { + "type": "string" + }, + "_acl": { + "items": { + "$ref": "#/components/schemas/Ace" + }, + "type": "array" + }, + "_name": { + "type": "string" + }, + "_encrypt": { + "items": { + "type": "string" + }, + "type": "array" + }, + "_version": { + "type": "number", + "format": "double" + }, + "Parameters": { + "items": { + "$ref": "#/components/schemas/WorkflowParameter" + }, + "type": "array" + }, + "background": { + "type": "boolean" + }, + "Serializable": { + "type": "boolean" + }, + "Filename": { + "type": "string" + }, + "projectandname": { + "type": "string" + }, + "Xaml": { + "type": "string" + } + }, + "required": [ + "_id", + "name", + "_created", + "_createdby", + "_createdbyid", + "_modified", + "_modifiedby", + "_modifiedbyid", + "_type", + "_acl", + "_name", + "_encrypt", + "_version", + "Parameters", + "background", + "Serializable", + "Filename", + "projectandname", + "Xaml" + ], + "type": "object", + "additionalProperties": false + }, + "Pick_Workflow.name-or-background-or-Serializable-or-Filename-or-projectandname-or-Xaml_": { + "properties": { + "name": { + "type": "string" + }, + "background": { + "type": "boolean" + }, + "Serializable": { + "type": "boolean" + }, + "Filename": { + "type": "string" + }, + "projectandname": { + "type": "string" + }, + "Xaml": { + "type": "string" + } + }, + "required": [ + "name", + "background", + "Serializable", + "Filename", + "projectandname", + "Xaml" + ], + "type": "object", + "description": "From T, pick a set of properties whose keys are in the union K" + }, + "OpenRPAWorkflowCreationParams": { + "$ref": "#/components/schemas/Pick_Workflow.name-or-background-or-Serializable-or-Filename-or-projectandname-or-Xaml_" + }, + "Collection": { + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "collection", + "timeseries" + ] + } + }, + "required": [ + "name", + "type" + ], + "type": "object", + "additionalProperties": false + }, + "CreateCollectionOptions": { + "properties": { + "timeseries": { + "properties": { + "metaField": { + "type": "string" + }, + "timeField": { + "type": "string" + } + }, + "type": "object" + }, + "expireAfterSeconds": { + "type": "number", + "format": "double" + }, + "name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "IRecordOfAny": { + "properties": {}, + "type": "object", + "additionalProperties": {} + }, + "CreateIndexOptions": { + "properties": { + "name": { + "type": "string" + }, + "options": { + "$ref": "#/components/schemas/IRecordOfAny" + }, + "keypath": { + "$ref": "#/components/schemas/IRecordOfAny" + } + }, + "required": [ + "keypath" + ], + "type": "object" + }, + "IStringToStringDictionary": { + "properties": {}, + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "QueryParams": { + "properties": { + "explain": { + "type": "boolean" + }, + "queryas": { + "type": "string" + }, + "sort": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/components/schemas/IStringToStringDictionary" + } + ] + }, + "skip": { + "type": "number", + "format": "double" + }, + "top": { + "type": "number", + "format": "double" + }, + "projection": { + "$ref": "#/components/schemas/IStringToStringDictionary" + }, + "query": {} + }, + "required": [ + "query" + ], + "type": "object" + }, + "AggregateParams": { + "properties": { + "explain": { + "type": "boolean" + }, + "queryas": { + "type": "string" + }, + "pipeline": {} + }, + "required": [ + "pipeline" + ], + "type": "object" + }, + "RunOpenRPAWorkflowParams": { + "properties": { + "rpc": { + "type": "boolean" + }, + "Parameters": { + "$ref": "#/components/schemas/IRecordOfAny" + }, + "workflowid": { + "type": "string" + } + }, + "required": [ + "rpc", + "workflowid" + ], + "type": "object" + }, + "QueueMessageParams": { + "properties": { + "rpc": { + "type": "boolean" + }, + "Payload": {} + }, + "required": [ + "rpc", + "Payload" + ], + "type": "object" + } + }, + "securitySchemes": { + "oidc": { + "type": "openIdConnect", + "openIdConnectUrl": "https://home.openiap.io/oidc/.well-known/openid-configuration" + } + } + }, + "info": { + "title": "@openiap/openflow", + "version": "1.5.7", + "description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)", + "license": { + "name": "MPL-2.0" + }, + "contact": { + "name": "OpenIAP ApS / Allan Zimmermann" + } + }, + "openapi": "3.0.0", + "paths": { + "/api/v1/me": { + "get": { + "operationId": "Me", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/User" + }, + "type": "array" + } + } + } + } + }, + "description": "Returns the current user", + "security": [ + { + "oidc": [] + } + ], + "parameters": [] + } + }, + "/api/v1/users": { + "get": { + "operationId": "GetUsers", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/User" + }, + "type": "array" + } + } + } + } + }, + "description": "Return a list of users ( max 100 by default)\neach users properties are limited to name, username, email, lastseen, Ude GetUser with _id to get all properties", + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "format": "double", + "type": "number" + } + }, + { + "in": "query", + "name": "projection", + "required": false, + "schema": { + "type": "string" + } + } + ] + }, + "post": { + "operationId": "CreateUser", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/User" + } + } + } + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UserCreationParams" + } + } + } + } + } + }, + "/api/v1/users/{id}": { + "get": { + "operationId": "GetUser", + "responses": { + "200": { + "description": "User", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/User" + } + } + } + } + }, + "description": "Return a user by _id", + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "description": "_id of user to return", + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string" + } + } + ] + }, + "delete": { + "operationId": "DeleteUser", + "responses": { + "204": { + "description": "No content" + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string" + } + } + ] + } + }, + "/api/v1/openrpaworkflows": { + "get": { + "operationId": "GetOpenrpaworkflows", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/Workflow" + }, + "type": "array" + } + } + } + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "format": "double", + "type": "number" + } + }, + { + "in": "query", + "name": "projection", + "required": false, + "schema": { + "type": "string" + } + } + ] + }, + "post": { + "operationId": "CreateOpenRPAWorkflow", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Workflow" + } + } + } + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/OpenRPAWorkflowCreationParams" + } + } + } + } + } + }, + "/api/v1/openrpaworkflows/{id}": { + "get": { + "operationId": "GetOpenRPAWorkflow", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Workflow" + } + } + } + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string" + } + } + ] + }, + "delete": { + "operationId": "DeleteOpenRPAWorkflow", + "responses": { + "204": { + "description": "No content" + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string" + } + } + ] + } + }, + "/api/v1/collections": { + "get": { + "operationId": "GetCollections", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/Collection" + }, + "type": "array" + } + } + } + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "query", + "name": "includesystem", + "required": false, + "schema": { + "type": "boolean" + } + } + ] + }, + "post": { + "operationId": "CreateCollection", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Collection" + } + } + } + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateCollectionOptions" + } + } + } + } + } + }, + "/api/v1/collections/{collectionname}": { + "get": { + "operationId": "GetCollection", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Collection" + } + } + } + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "path", + "name": "collectionname", + "required": true, + "schema": { + "type": "string" + } + } + ] + }, + "delete": { + "operationId": "DropCollection", + "responses": { + "204": { + "description": "No content" + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "path", + "name": "collectionname", + "required": true, + "schema": { + "type": "string" + } + } + ] + } + }, + "/api/v1/indexes/{collectionname}": { + "get": { + "operationId": "GetIndexes", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "items": {}, + "type": "array" + } + } + } + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "path", + "name": "collectionname", + "required": true, + "schema": { + "type": "string" + } + } + ] + }, + "post": { + "operationId": "CreateIndex", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "type": "string" + } + } + } + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "path", + "name": "collectionname", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateIndexOptions" + } + } + } + } + } + }, + "/api/v1/indexes/{collectionname}/{name}": { + "delete": { + "operationId": "DropIndex", + "responses": { + "204": { + "description": "No content" + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "path", + "name": "collectionname", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ] + } + }, + "/api/v1/query/{collectionname}": { + "post": { + "operationId": "Query", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "items": {}, + "type": "array" + } + } + } + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "path", + "name": "collectionname", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/QueryParams" + } + } + } + } + } + }, + "/api/v1/aggregate/{collectionname}": { + "post": { + "operationId": "Aggregate", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "items": {}, + "type": "array" + } + } + } + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "path", + "name": "collectionname", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AggregateParams" + } + } + } + } + } + }, + "/api/v1/runopenrpaworkflow/{robotname}": { + "post": { + "operationId": "RunOpenRPAWorkflow", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IRecordOfAny" + } + } + } + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "path", + "name": "robotname", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RunOpenRPAWorkflowParams" + } + } + } + } + } + }, + "/api/v1/queuemessage/{queuename}": { + "post": { + "operationId": "QueueMessage", + "responses": { + "200": { + "description": "Ok", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IRecordOfAny" + } + } + } + } + }, + "security": [ + { + "oidc": [] + } + ], + "parameters": [ + { + "in": "path", + "name": "queuename", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/QueueMessageParams" + } + } + } + } + } + } + }, + "servers": [ + { + "url": "/" + } + ] +} \ No newline at end of file diff --git a/OpenFlow/tsconfig.json b/OpenFlow/tsconfig.json index 9927ad74..a6610c12 100644 --- a/OpenFlow/tsconfig.json +++ b/OpenFlow/tsconfig.json @@ -7,6 +7,7 @@ "target": "es2018", "outDir": "../dist", "rootDir": "src", + "resolveJsonModule": true, "sourceMap": true, "declaration": true, "lib": [ diff --git a/gulpfile.js b/gulpfile.js index 34f99c5e..e49d999f 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -19,7 +19,8 @@ let watch = false; const OpenFlowPublicFiles = [ "./OpenFlow/src/public/**/*.html", "./OpenFlow/src/public/**/*.css", "./OpenFlow/src/public/**/*.js", "./OpenFlow/src/public/**/*.json", "./OpenFlow/src/public/**/*.ico", "./OpenFlow/src/public/**/*.eot", "./OpenFlow/src/public/**/*.svg", "./OpenFlow/src/public/**/*.ttf", - "./OpenFlow/src/public/**/*.woff", "./OpenFlow/src/public/**/*.woff2", "./OpenFlow/src/public/**/*.png", "./OpenFlow/src/public/**/*.lang"]; + "./OpenFlow/src/public/**/*.woff", "./OpenFlow/src/public/**/*.woff2", "./OpenFlow/src/public/**/*.png", "./OpenFlow/src/public/**/*.lang", + "./OpenFlow/src/public/**/*.json"]; const NodeREDHTMLFiles = ["./OpenFlowNodeRED/src/nodered/nodes/**/*.html", "./OpenFlowNodeRED/src/nodered/nodes/**/*.png", "./OpenFlowNodeRED/src/nodered/nodes/**/*.json"] const publicdestination = "./dist/public"; @@ -238,32 +239,31 @@ gulp.task("browserify", function () { // `docker buildx build -t openiap/nodered-puppeteer:edge -t openiap/nodered-puppeteer:` + version + ` --platform linux/amd64 --push -f ./OpenFlowNodeRED/Dockerfilepuppeteer .`, // `docker buildx build -t openiap/nodered-tagui:edge -t openiap/nodered-tagui:` + version + ` --platform linux/amd64 --push -f ./OpenFlowNodeRED/Dockerfiletagui .`, // ])); -gulp.task("compose", shell.task([ - // docker buildx create --name openiap --use - // linux workaround ? - // docker run -it --rm --privileged tonistiigi/binfmt --install all - // docker buildx use default - // docker buildx build --platform linux/amd64 -t openiap/openflow:edge . - // second work around - // wget https://github.com/docker/buildx/releases/download/v0.4.1/buildx-v0.4.1.linux-amd64 - // chmod a+x buildx-v0.4.1.linux-amd64 - // mkdir -p ~/.docker/cli-plugins - // mv buildx-v0.4.1.linux-amd64 ~/.docker/cli-plugins/docker-buildx - - `docker buildx build -t openiap/openflow:edge -t openiap/openflow:` + version + ` --platform linux/amd64 --push .`, - // `docker buildx build -t openiap/nodered:edge -t openiap/nodered:` + version + ` --platform linux/amd64 --push --file ./OpenFlowNodeRED/Dockerfile .`, - - - // `echo "docker buildx build -t openiap/openflow:edge -t openiap/openflow:` + version + ` --platform linux/amd64 --push ."`, - // `echo "docker buildx build -t openiap/nodered:edge -t openiap/nodered:` + version + ` --platform linux/amd64 --push --file ./OpenFlowNodeRED/Dockerfile ."`, - // `echo "docker buildx build -t openiap/nodered-puppeteer:edge -t openiap/nodered-puppeteer:` + version + ` --platform linux/amd64 --push -f ./OpenFlowNodeRED/Dockerfilepuppeteer ."`, - // `echo "docker buildx build -t openiap/nodered-tagui:edge -t openiap/nodered-tagui:` + version + ` --platform linux/amd64 --push -f ./OpenFlowNodeRED/Dockerfiletagui ."`, - // `docker buildx build -t openiap/openflow:edge -t openiap/openflow:` + version + ` --platform linux/amd64 --push .`, - // `docker buildx build -t openiap/nodered:edge -t openiap/nodered:` + version + ` --platform linux/amd64 --push --file ./OpenFlowNodeRED/Dockerfile .`, - // `docker buildx build -t openiap/nodered-puppeteer:edge -t openiap/nodered-puppeteer:` + version + ` --platform linux/amd64 --push -f ./OpenFlowNodeRED/Dockerfilepuppeteer .`, - // `docker buildx build -t openiap/nodered-tagui:edge -t openiap/nodered-tagui:` + version + ` --platform linux/amd64 --push -f ./OpenFlowNodeRED/Dockerfiletagui .`, -])); - +gulp.task("compose", async function (done) { + var versions = ["-t openiap/openflow:edge", "-t openiap/openflow:" + version] + const dotCount = version.split('.').length - 1; + if(dotCount == 3){ + majorversion = version.substring(0, version.lastIndexOf('.')); + versions.push("-t openiap/openflow:" + majorversion); + } + console.log(versions) + return shell.task([`docker buildx build ${versions.join(" ")} --platform linux/amd64 --push .`])(); +}); +gulp.task("latest", async function (done) { + var versions = ["-t openiap/openflow:latest", "-t openiap/openflow:edge", "-t openiap/openflow:" + version] + const dotCount = version.split('.').length - 1; + if(dotCount == 3){ + majorversion = version.substring(0, version.lastIndexOf('.')); + versions.push("-t openiap/openflow:" + majorversion); + } + console.log(versions) + return shell.task([`docker buildx build ${versions.join(" ")} --platform linux/amd64 --push .`])(); +}); + +/* +docker buildx build -t openiap/openflow:edge -t openiap/openflow:latest -t openiap/openflow:1.5.0 --platform linux/amd64 --push . +docker buildx build -t openiap/openflow:edge -t openiap/openflow:latest -t openiap/openflow:1.5.0 --platform linux/arm/v7 --push . +docker buildx build -t openiap/openflow:edge -t openiap/openflow:latest -t openiap/openflow:1.5.0 --platform linux/arm/v6 --push . gulp.task("latest", shell.task([ `echo "docker buildx build -t openiap/openflow:edge -t openiap/openflow:latest -t openiap/openflow:` + version + ` --platform linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 --push ."`, // `echo "docker buildx build -t openiap/nodered:edge -t openiap/nodered:latest -t openiap/nodered:` + version + ` --platform linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6 --push --file ./OpenFlowNodeRED/Dockerfile ."`, @@ -275,10 +275,6 @@ gulp.task("latest", shell.task([ // `docker buildx build -t openiap/nodered-puppeteer:edge -t openiap/nodered-puppeteer:latest -t openiap/nodered-puppeteer:` + version + ` --platform linux/amd64 --push -f ./OpenFlowNodeRED/Dockerfilepuppeteer .`, // `docker buildx build -t openiap/nodered-tagui:edge -t openiap/nodered-tagui:latest -t openiap/nodered-tagui:` + version + ` --platform linux/amd64 --push -f ./OpenFlowNodeRED/Dockerfiletagui .`, ])); -/* -docker buildx build -t openiap/openflow:edge -t openiap/openflow:latest -t openiap/openflow:1.5.0 --platform linux/amd64 --push . -docker buildx build -t openiap/openflow:edge -t openiap/openflow:latest -t openiap/openflow:1.5.0 --platform linux/arm/v7 --push . -docker buildx build -t openiap/openflow:edge -t openiap/openflow:latest -t openiap/openflow:1.5.0 --platform linux/arm/v6 --push . */ diff --git a/package.json b/package.json index 870bc94c..104c0d70 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@openiap/openflow", - "version": "1.5.6", - "description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)", + "version": "1.5.7.20", + "description": "Simple wrapper around NodeRed, RabbitMQ and MongoDB to support a more scaleable NodeRed implementation.\r Also the \"backend\" for [OpenRPA](https://github.com/skadefro/OpenRPA)", "main": "index.js", "scripts": { "test": "_mocha 'test/**/*.test.ts'", @@ -15,7 +15,10 @@ "build": "docker-compose up registry -d && docker-compose push --ignore-push-failures && docker-compose up -d", "updateapilocal": "npm uninstall @openiap/nodeapi && npm i ../nodeapi", "updateapidev": "npm uninstall @openiap/nodeapi && npm i openiap/nodeapi", - "updateapi": "npm uninstall @openiap/nodeapi && npm i @openiap/nodeapi" + "updateapi": "npm uninstall @openiap/nodeapi && npm i @openiap/nodeapi", + "spec": "tsoa spec", + "routes": "tsoa routes", + "generate": "tsoa spec && tsoa routes" }, "repository": { "type": "git", @@ -35,7 +38,7 @@ "@fortawesome/fontawesome-free": "^5.15.3", "@kubernetes/client-node": "^0.19.0", "@node-saml/passport-saml": "^4.0.1", - "@openiap/nodeapi": "^0.0.30", + "@openiap/nodeapi": "^0.0.31", "@openiap/openflow-api": "^2.1.11", "@opentelemetry/exporter-metrics-otlp-grpc": "^0.43.0", "@opentelemetry/sdk-node": "^0.43.0", @@ -80,9 +83,11 @@ "request": "^2.88.2", "saml20": "^0.1.14", "samlp": "^6.0.2", + "showdown": "^2.1.0", "swagger-autogen": "^2.23.1", "swagger-ui-express": "^4.6.3", "systeminformation": "^5.21.9", + "tsoa": "^5.1.1", "web-push": "^3.5.0", "ws": "^8.11.0", "xml2js": "^0.4.23" diff --git a/tsoa.json b/tsoa.json new file mode 100644 index 00000000..c9f968be --- /dev/null +++ b/tsoa.json @@ -0,0 +1,21 @@ +{ + "entryFile": "OpenFlow/src/ee/OpenAPIProxy.ts", + "noImplicitAdditionalProperties": "throw-on-extras", + "controllerPathGlobs2": ["src/**/*Controller.ts"], + "controllerPathGlobs": ["OpenFlow/src/ee/OpenAPIuser.ts"], + "spec": { + "outputDirectory": "OpenFlow/src/public", + "specVersion": 3, + "securityDefinitions": { + "oidc": { + "type": "openIdConnect", + "openIdConnectUrl": "https://app.openiap.io/oidc/.well-known/openid-configuration" + } + + } + }, + "routes": { + "routesDir": "OpenFlow/src/ee/build", + "authenticationModule": "OpenFlow/src/ee/OpenAPIauthentication.ts" + } + } \ No newline at end of file