From 23cc2f81e9f07f4945b91926fc2365c9b44217d9 Mon Sep 17 00:00:00 2001 From: archer <545436317@qq.com> Date: Fri, 1 Sep 2023 11:47:06 +0800 Subject: [PATCH] perf: code --- client/public/locales/zh/common.json | 2 +- client/src/pages/api/system/getInitData.ts | 2 ++ client/src/pages/kb/detail/index.tsx | 6 +++- client/src/service/events/pushBill.ts | 6 ++-- client/src/service/mongo.ts | 30 ++---------------- client/src/service/pg.ts | 26 ++++++++++++++++ client/src/service/response.ts | 36 ++-------------------- client/src/service/utils/tools.ts | 21 +++++++++++-- 8 files changed, 61 insertions(+), 68 deletions(-) diff --git a/client/public/locales/zh/common.json b/client/public/locales/zh/common.json index 96b3c3edb45..95febf156fe 100644 --- a/client/public/locales/zh/common.json +++ b/client/public/locales/zh/common.json @@ -83,7 +83,7 @@ }, "dataset": { "Confirm to delete the data": "确认删除该数据？", - "Queue Desc": "该数据是指整个系统当前待训练的数量。FastGPT 采用排队训练的方式，如果待训练的数据过多，可能需要等待一段时间", + "Queue Desc": "该数据是指整个系统当前待训练的数量。{{title}} 采用排队训练的方式，如果待训练的数据过多，可能需要等待一段时间", "System Data Queue": "排队长度" }, "file": { diff --git a/client/src/pages/api/system/getInitData.ts b/client/src/pages/api/system/getInitData.ts index 69410091229..7a95d619cc9 100644 --- a/client/src/pages/api/system/getInitData.ts +++ b/client/src/pages/api/system/getInitData.ts @@ -88,6 +88,8 @@ const defaultVectorModels: VectorModelItemType[] = [ export async function getInitConfig() { try { + if (global.feConfigs) return; + const filename = process.env.NODE_ENV === 'development' ? 'data/config.local.json' : '/app/data/config.json'; const res = JSON.parse(readFileSync(filename, 'utf-8')); diff --git a/client/src/pages/kb/detail/index.tsx b/client/src/pages/kb/detail/index.tsx index 43e63a4a6ef..c13d6836ce6 100644 --- a/client/src/pages/kb/detail/index.tsx +++ b/client/src/pages/kb/detail/index.tsx @@ -22,6 +22,7 @@ import { useTranslation } from 'react-i18next'; import { getTrainingQueueLen } from '@/api/plugins/kb'; import MyTooltip from '@/components/MyTooltip'; import { QuestionOutlineIcon } from '@chakra-ui/icons'; +import { feConfigs } from '@/store/static'; const ImportData = dynamic(() => import('./components/Import'), { ssr: false @@ -119,7 +120,10 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` } {t('dataset.System Data Queue')} - + diff --git a/client/src/service/events/pushBill.ts b/client/src/service/events/pushBill.ts index 83a2a53d614..853c3b89540 100644 --- a/client/src/service/events/pushBill.ts +++ b/client/src/service/events/pushBill.ts @@ -72,7 +72,7 @@ export const updateShareChatBill = async ({ } ); } catch (err) { - addLog.error('update shareChat error', { err }); + addLog.error('update shareChat error', err); } }; @@ -111,7 +111,7 @@ export const pushQABill = async ({ $inc: { balance: -total } }); } catch (err) { - addLog.error('Create completions bill error', { err }); + addLog.error('Create completions bill error', err); billId && Bill.findByIdAndDelete(billId); } }; @@ -160,7 +160,7 @@ export const pushGenerateVectorBill = async ({ $inc: { balance: -total } }); } catch (err) { - addLog.error('Create generateVector bill error', { err }); + addLog.error('Create generateVector bill error', err); billId && Bill.findByIdAndDelete(billId); } } catch (error) { diff --git a/client/src/service/mongo.ts b/client/src/service/mongo.ts index 5ac3bcbc7b7..f623d6208e0 100644 --- a/client/src/service/mongo.ts +++ b/client/src/service/mongo.ts @@ -4,9 +4,8 @@ import { startQueue } from './utils/tools'; import { getInitConfig } from '@/pages/api/system/getInitData'; import { User } from './models/user'; import { PRICE_SCALE } from '@/constants/common'; -import { connectPg, PgClient } from './pg'; +import { initPg } from './pg'; import { createHashPassword } from '@/utils/tools'; -import { PgTrainingTableName } from '@/constants/plugin'; import { createLogger, format, transports } from 'winston'; import 'winston-mongodb'; @@ -81,9 +80,9 @@ function initLogger() { format.printf((info) => { if (info.level === 'error') { console.log(info.meta); - return `${info.level}: ${[info.timestamp]}: ${info.message}`; + return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}`; } - return `${info.level}: ${[info.timestamp]}: ${info.message}${ + return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}${ info.meta ? `: ${JSON.stringify(info.meta)}` : '' }`; }) @@ -123,29 +122,6 @@ async function initRootUser() { console.log('init root user error', error); } } -async function initPg() { - try { - await connectPg(); - await PgClient.query(` - CREATE EXTENSION IF NOT EXISTS vector; - CREATE TABLE IF NOT EXISTS ${PgTrainingTableName} ( - id BIGSERIAL PRIMARY KEY, - vector VECTOR(1536) NOT NULL, - user_id VARCHAR(50) NOT NULL, - kb_id VARCHAR(50) NOT NULL, - source VARCHAR(100), - q TEXT NOT NULL, - a TEXT NOT NULL - ); - CREATE INDEX IF NOT EXISTS modelData_userId_index ON ${PgTrainingTableName} USING HASH (user_id); - CREATE INDEX IF NOT EXISTS modelData_kbId_index ON ${PgTrainingTableName} USING HASH (kb_id); - CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON ${PgTrainingTableName} (md5(q), md5(a), user_id, kb_id); - `); - console.log('init pg successful'); - } catch (error) { - console.log('init pg error', error); - } -} export * from './models/chat'; export * from './models/chatItem'; diff --git a/client/src/service/pg.ts b/client/src/service/pg.ts index 723f019343d..61d2ff10775 100644 --- a/client/src/service/pg.ts +++ b/client/src/service/pg.ts @@ -1,6 +1,8 @@ import { Pool } from 'pg'; import type { QueryResultRow } from 'pg'; import { PgTrainingTableName } from '@/constants/plugin'; +import { exit } from 'process'; +import { addLog } from './utils/tools'; export const connectPg = async (): Promise => { if (global.pgClient) { @@ -184,3 +186,27 @@ export const insertKbItem = ({ ]) }); }; + +export async function initPg() { + try { + await connectPg(); + await PgClient.query(` + CREATE EXTENSION IF NOT EXISTS vector; + CREATE TABLE IF NOT EXISTS ${PgTrainingTableName} ( + id BIGSERIAL PRIMARY KEY, + vector VECTOR(1536) NOT NULL, + user_id VARCHAR(50) NOT NULL, + kb_id VARCHAR(50) NOT NULL, + source VARCHAR(100), + q TEXT NOT NULL, + a TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS modelData_userId_index ON ${PgTrainingTableName} USING HASH (user_id); + CREATE INDEX IF NOT EXISTS modelData_kbId_index ON ${PgTrainingTableName} USING HASH (kb_id); + CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON ${PgTrainingTableName} (md5(q), md5(a), user_id, kb_id); + `); + console.log('init pg successful'); + } catch (error) { + addLog.error('init pg error', error); + } +} diff --git a/client/src/service/response.ts b/client/src/service/response.ts index a77bb14ae13..830a4bb6ab2 100644 --- a/client/src/service/response.ts +++ b/client/src/service/response.ts @@ -53,23 +53,7 @@ export const jsonRes = ( msg = openaiError[error.response.statusText]; } - addLog.error(msg, { - message: msg, - stack: error?.stack, - ...(error?.config && { - config: { - headers: error.config.headers, - url: error.config.url, - data: error.config.data - } - }), - ...(error?.response && { - response: { - status: error.response.status, - statusText: error.response.statusText - } - }) - }); + addLog.error(`response error: ${msg}`, error); } res.status(code).json({ @@ -110,23 +94,7 @@ export const sseErrRes = (res: NextApiResponse, error: any) => { msg = openaiError[error.response.statusText]; } - addLog.error(`sse error: ${msg}`, { - message: msg, - stack: error?.stack, - ...(error?.config && { - config: { - headers: error.config.headers, - url: error.config.url, - data: error.config.data - } - }), - ...(error?.response && { - response: { - status: error.response.status, - statusText: error.response.statusText - } - }) - }); + addLog.error(`sse error: ${msg}`, error); sseResponse({ res, diff --git a/client/src/service/utils/tools.ts b/client/src/service/utils/tools.ts index 423b9a9113e..ce8e206e0c5 100644 --- a/client/src/service/utils/tools.ts +++ b/client/src/service/utils/tools.ts @@ -94,7 +94,24 @@ export const addLog = { info: (msg: string, obj?: Record) => { global.logger?.info(msg, { meta: obj }); }, - error: (msg: string, obj?: Record) => { - global.logger?.error(msg, { meta: obj }); + error: (msg: string, error?: any) => { + global.logger?.error(msg, { + meta: { + stack: error?.stack, + ...(error?.config && { + config: { + headers: error.config.headers, + url: error.config.url, + data: error.config.data + } + }), + ...(error?.response && { + response: { + status: error.response.status, + statusText: error.response.statusText + } + }) + } + }); } };