4.6.7 fix (#752)

labring · Jan 19, 2024 · 5e2adb2 · 5e2adb2
1 parent c031e6d
commit 5e2adb2
Show file tree

Hide file tree

Showing 37 changed files with 420 additions and 293 deletions.
diff --git a/docSite/content/docs/development/docker.md b/docSite/content/docs/development/docker.md
@@ -17,6 +17,11 @@ weight: 707
 | 500w 组向量 | 8c32g | 16c64g 200GB |
 {{< /table >}}
 
+## 部署架构图
+
+![](/imgs/sealos-fastgpt.webp)
+
+
 ### 1. 准备好代理环境（国外服务器可忽略）
 
 确保可以访问 OpenAI，具体方案可以参考：[代理方案](/docs/development/proxy/)。或直接在 Sealos 上 [部署 OneAPI](/docs/development/one-api)，既解决代理问题也能实现多 Key 轮询、接入其他大模型。

diff --git a/docSite/content/docs/development/qa.md b/docSite/content/docs/development/qa.md
@@ -19,6 +19,10 @@ images: []
 
 ## 通用问题
 
+### 能否纯本地允许
+
+可以。需要准备好向量模型和LLM模型。
+
 ### insufficient_user_quota user quota is not enough 
 
 OneAPI 账号的余额不足，默认 root 用户只有 200 刀，可以手动修改。
@@ -105,7 +109,7 @@ mongo连接失败，检查
 
 ### TypeError: Cannot read properties of null (reading 'useMemo' )
 
-用 Node18 试试，可能最新的 Node 有问题。 本地开发流程：
+删除所有的`node_modules`，用 Node18 重新 install 试试，可能最新的 Node 有问题。 本地开发流程：
 
 1. 根目录: `pnpm i`
 2. 复制 `config.json` -> `config.local.json`

diff --git a/packages/global/common/file/icon.ts b/packages/global/common/file/icon.ts
@@ -3,10 +3,12 @@ export const fileImgs = [
   { suffix: 'csv', src: 'file/fill/csv' },
   { suffix: '(doc|docs)', src: 'file/fill/doc' },
   { suffix: 'txt', src: 'file/fill/txt' },
-  { suffix: 'md', src: 'file/fill/markdown' }
+  { suffix: 'md', src: 'file/fill/markdown' },
+  { suffix: 'html', src: 'file/fill/html' }
+
   // { suffix: '.', src: '/imgs/files/file.svg' }
 ];
 
-export function getFileIcon(name = '', defaultImg = '/imgs/files/file.svg') {
+export function getFileIcon(name = '', defaultImg = 'file/fill/file') {
   return fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(name))?.src || defaultImg;
 }
diff --git a/packages/global/common/string/markdown.ts b/packages/global/common/string/markdown.ts
@@ -51,19 +51,18 @@ export const uploadMarkdownBase64 = async ({
     // match base64, upload and replace it
     const base64Regex = /data:image\/.*;base64,([^\)]+)/g;
     const base64Arr = rawText.match(base64Regex) || [];
+
     // upload base64 and replace it
-    await Promise.all(
-      base64Arr.map(async (base64Img) => {
-        try {
-          const str = await uploadImgController(base64Img);
+    for await (const base64Img of base64Arr) {
+      try {
+        const str = await uploadImgController(base64Img);
 
-          rawText = rawText.replace(base64Img, str);
-        } catch (error) {
-          rawText = rawText.replace(base64Img, '');
-          rawText = rawText.replace(/!\[.*\]\(\)/g, '');
-        }
-      })
-    );
+        rawText = rawText.replace(base64Img, str);
+      } catch (error) {
+        rawText = rawText.replace(base64Img, '');
+        rawText = rawText.replace(/!\[.*\]\(\)/g, '');
+      }
+    }
   }
 
   // Remove white space on both sides of the picture

diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts
@@ -48,10 +48,6 @@ export type FileCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams
   name: string;
   rawTextLength: number;
   hashRawText: string;
-  trainingType: `${TrainingModeEnum}`;
-  chunkSize: number;
-  chunkSplitter: string;
-  qaPrompt: string;
 
   fileMetadata?: Record<string, any>;
   collectionMetadata?: Record<string, any>;
@@ -74,3 +70,14 @@ export type PostWebsiteSyncParams = {
   datasetId: string;
   billId: string;
 };
+
+export type PushDatasetDataProps = {
+  collectionId: string;
+  data: PushDatasetDataChunkProps[];
+  trainingMode: `${TrainingModeEnum}`;
+  prompt?: string;
+  billId?: string;
+};
+export type PushDatasetDataResponse = {
+  insertLen: number;
+};
diff --git a/packages/global/core/dataset/controller.d.ts b/packages/global/core/dataset/controller.d.ts
@@ -21,7 +21,7 @@ export type UpdateDatasetDataProps = {
 };
 
 export type PatchIndexesProps = {
-  type: 'create' | 'update' | 'delete';
+  type: 'create' | 'update' | 'delete' | 'unChange';
   index: Omit<DatasetDataIndexItemType, 'dataId'> & {
     dataId?: string;
   };

diff --git a/packages/service/common/file/image/controller.ts b/packages/service/common/file/image/controller.ts
@@ -46,8 +46,17 @@ export async function readMongoImg({ id }: { id: string }) {
   return data?.binary;
 }
 
-export async function delImgByRelatedId(relateIds: string[]) {
+export async function delImgByRelatedId({
+  teamId,
+  relateIds
+}: {
+  teamId: string;
+  relateIds: string[];
+}) {
+  if (relateIds.length === 0) return;
+
   return MongoImage.deleteMany({
+    teamId,
     'metadata.relatedId': { $in: relateIds.map((id) => String(id)) }
   });
 }
diff --git a/packages/service/common/file/image/schema.ts b/packages/service/common/file/image/schema.ts
@@ -34,9 +34,8 @@ const ImageSchema = new Schema({
 try {
   ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 });
   ImageSchema.index({ type: 1 });
-  ImageSchema.index({ teamId: 1 });
   ImageSchema.index({ createTime: 1 });
-  ImageSchema.index({ 'metadata.relatedId': 1 });
+  ImageSchema.index({ teamId: 1, 'metadata.relatedId': 1 });
 } catch (error) {
   console.log(error);
 }

diff --git a/packages/service/common/file/multer.ts b/packages/service/common/file/multer.ts
@@ -28,12 +28,16 @@ export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => {
         // },
         filename: async (req, file, cb) => {
           const { ext } = path.parse(decodeURIComponent(file.originalname));
-          cb(null, `${getNanoid(32)}${ext}`);
+          cb(null, `${getNanoid()}${ext}`);
         }
       })
     }).single('file');
 
-    async doUpload<T = Record<string, any>>(req: NextApiRequest, res: NextApiResponse) {
+    async doUpload<T = Record<string, any>>(
+      req: NextApiRequest,
+      res: NextApiResponse,
+      originBuckerName?: `${BucketNameEnum}`
+    ) {
       return new Promise<{
         file: FileType;
         metadata: Record<string, any>;
@@ -47,7 +51,7 @@ export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => {
           }
 
           // check bucket name
-          const bucketName = req.body?.bucketName as `${BucketNameEnum}`;
+          const bucketName = (req.body?.bucketName || originBuckerName) as `${BucketNameEnum}`;
           if (bucketName && !bucketNameMap[bucketName]) {
             return reject('BucketName is invalid');
           }

diff --git a/packages/service/common/vectorStore/pg/controller.ts b/packages/service/common/vectorStore/pg/controller.ts
@@ -39,14 +39,15 @@ export const insertDatasetDataVector = async (
   }
 ): Promise<{ insertId: string }> => {
   const { teamId, datasetId, collectionId, vectors, retry = 3 } = props;
+
   try {
     const { rows } = await PgClient.insert(PgDatasetTableName, {
       values: [
         [
           { key: 'vector', value: `[${vectors[0]}]` },
           { key: 'team_id', value: String(teamId) },
-          { key: 'dataset_id', value: datasetId },
-          { key: 'collection_id', value: collectionId }
+          { key: 'dataset_id', value: String(datasetId) },
+          { key: 'collection_id', value: String(collectionId) }
         ]
       ]
     });
@@ -176,8 +177,8 @@ export const getVectorDataByTime = async (start: Date, end: Date) => {
   `);
 
   return rows.map((item) => ({
-    id: item.id,
-    datasetId: item.dataset_id,
-    teamId: item.team_id
+    id: String(item.id),
+    teamId: item.team_id,
+    datasetId: item.dataset_id
   }));
 };
diff --git a/packages/service/core/chat/chatItemSchema.ts b/packages/service/core/chat/chatItemSchema.ts
@@ -89,6 +89,7 @@ try {
      close custom feedback; 
   */
   ChatItemSchema.index({ appId: 1, chatId: 1, dataId: 1 }, { background: true });
+  ChatItemSchema.index({ time: -1 }, { background: true });
   ChatItemSchema.index({ userGoodFeedback: 1 }, { background: true });
   ChatItemSchema.index({ userBadFeedback: 1 }, { background: true });
   ChatItemSchema.index({ customFeedbacks: 1 }, { background: true });

diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts
@@ -25,7 +25,7 @@ export async function createOneCollection({
   type,
 
   trainingType = TrainingModeEnum.chunk,
-  chunkSize = 0,
+  chunkSize = 512,
   chunkSplitter,
   qaPrompt,
 
@@ -134,7 +134,10 @@ export async function delCollectionAndRelatedSources({
 
   // delete file and imgs
   await Promise.all([
-    delImgByRelatedId(relatedImageIds),
+    delImgByRelatedId({
+      teamId,
+      relateIds: relatedImageIds
+    }),
     delFileByFileIdList({
       bucketName: BucketNameEnum.dataset,
       fileIdList