Merge pull request #20 from smalltong02/RAG_dev1.0.1

support code models.

WebUI/Server/utils.py

@@ -229,7 +229,7 @@ def get_model_worker_config(model_name: str = None) -> dict:
             return config
 
     modelinfo["mtype"], modelinfo["msize"], modelinfo["msubtype"] = GetModelInfoByName(webui_config, model_name)
-    if modelinfo["mtype"] == ModelType.Local or modelinfo["mtype"] == ModelType.Multimodal or modelinfo["mtype"] == ModelType.Special:
+    if modelinfo["mtype"] == ModelType.Local or modelinfo["mtype"] == ModelType.Multimodal or modelinfo["mtype"] == ModelType.Code or modelinfo["mtype"] == ModelType.Special:
         modelinfo["mname"] = model_name
         modelinfo["config"] = GetModelConfig(webui_config, modelinfo)
         if modelinfo["config"]:
@@ -239,6 +239,8 @@ def get_model_worker_config(model_name: str = None) -> dict:
             config["max_gpu_memory"] = get_max_gpumem(modelinfo["config"])
         if modelinfo["mtype"] == ModelType.Special:
             config["special_model"] = True
+        if modelinfo["mtype"] == ModelType.Code:
+            config["code_model"] = True
         if modelinfo["mtype"] == ModelType.Multimodal:
             config["multimodal_model"] = True
     return config

WebUI/configs/basicconfig.py

@@ -13,16 +13,17 @@
 if not TMP_DIR.exists():
     TMP_DIR.mkdir(exist_ok=True, parents=True)
 
-glob_model_type_list = ["LLM Model","Multimodal Model","Special Model","Online Model"]
+glob_model_type_list = ["LLM Model","Multimodal Model", "Code Model", "Special Model","Online Model"]
 glob_model_size_list = ["3B Model","7B Model","13B Model","34B Model","70B Model"]
 glob_model_subtype_list = ["Vision Chat Model","Voice Chat Model","Video Chat Model"]
 
 class ModelType(Enum):
     Unknown = 0
     Local = 1
     Multimodal = 2
-    Special = 3
-    Online = 4
+    Code = 3
+    Special = 4
+    Online = 5
 
 class ModelSize(Enum):
     Unknown = 0
@@ -51,6 +52,8 @@ def GetTypeName(type: ModelType) -> str:
         return "LLM Model"
     if type == ModelType.Multimodal:
         return "Multimodal Model"
+    if type == ModelType.Code:
+        return "Code Model"
     if type == ModelType.Special:
         return "Special Model"
     if type == ModelType.Online:
@@ -84,6 +87,8 @@ def GetModelType(Typestr : str) -> ModelType:
         return ModelType.Local
     if Typestr == "Multimodal Model":
         return ModelType.Multimodal
+    if Typestr == "Code Model":
+        return ModelType.Code
     if Typestr == "Special Model":
         return ModelType.Special
     if Typestr == "Online Model":
@@ -165,6 +170,9 @@ def GetModeList(webui_config, current_model  : Dict[str, any]) -> list:
     if mtype == ModelType.Local:
         msize = current_model["msize"]
         return [f"{key}" for key in localmodel.get("LLM Model").get(GetSizeName(msize))]
+    if mtype == ModelType.Code:
+        msize = current_model["msize"]
+        return [f"{key}" for key in localmodel.get("Code Model").get(GetSizeName(msize))]
     elif mtype == ModelType.Special:
         msize = current_model["msize"]
         return [f"{key}" for key in localmodel.get("Special Model").get(GetSizeName(msize))]
@@ -180,6 +188,9 @@ def GetModelConfig(webui_config, current_model : Dict[str, any]) -> Dict:
     if mtype == ModelType.Local:
         msize = GetSizeName(current_model["msize"])
         provider = "LLM Model"
+    elif mtype == ModelType.Code:
+        msize = GetSizeName(current_model["msize"])
+        provider = "Code Model"
     elif mtype == ModelType.Special:
         msize = GetSizeName(current_model["msize"])
         provider = "Special Model"

WebUI/configs/specialmodels.py

@@ -6,6 +6,7 @@
 import google.generativeai as genai
 from fastapi.responses import StreamingResponse
 from WebUI.configs.basicconfig import *
+from WebUI.configs.codemodels import *
 from WebUI.configs.webuiconfig import InnerJsonConfigWebUIParse
 from WebUI.Server.db.repository import add_chat_history_to_db, update_chat_history
 from WebUI.Server.chat.StreamHandler import StreamSpeakHandler
@@ -51,7 +52,7 @@ def load_pipeline_model(app: FastAPI, model_name, model_path, device):
         "text-generation",
         model=model,
         tokenizer=tokenizer,
-        max_length=512,
+        max_length=1024,
         do_sample=True,
         temperature=0.7,
         top_p=0.95,
@@ -77,7 +78,7 @@ def load_llamacpp_model(app: FastAPI, model_name, model_path):
             model_path=path,
             do_sample=True,
             temperature=0.7,
-            max_tokens=512,
+            max_tokens=1024,
             top_p=1,
             verbose=True,
             callback_manager=callback_manager,
@@ -504,5 +505,7 @@ def model_chat(
     modelinfo["mname"] = model_name
     if modelinfo["mtype"] == ModelType.Special or modelinfo["mtype"] == ModelType.Online:
         return special_model_chat(app._model, app._streamer, modelinfo, query, imagesdata, audiosdata, videosdata, imagesprompt, history, stream, speechmodel, temperature, max_tokens, prompt_name)
+    elif modelinfo["mtype"] == ModelType.Code:
+        return code_model_chat(app._model, app._tokenizer, app._streamer, modelinfo, query, imagesdata, audiosdata, videosdata, imagesprompt, history, False, speechmodel, temperature, max_tokens, prompt_name)
     elif modelinfo["mtype"] == ModelType.Multimodal:
         return multimodal_model_chat(app._model, app._tokenizer, modelinfo, query, imagesdata, audiosdata, videosdata, imagesprompt, history, False, speechmodel, temperature, max_tokens, prompt_name)

WebUI/configs/webuiconfig.json

@@ -137,6 +137,24 @@
                     "cpu_offload": false,
                     "refiner": true
                 }
+            },
+            "stable-video-diffusion-img2vid": {
+                "type": "local",
+                "path": "models/imagegeneration/stable-video-diffusion-img2vid",
+                "device": "auto",
+                "loadbits": 16,
+                "Huggingface": "stabilityai/stable-video-diffusion-img2vid",
+                "config": {
+                }
+            },
+            "stable-video-diffusion-img2vid-xt": {
+                "type": "local",
+                "path": "models/imagegeneration/stable-video-diffusion-img2vid-xt",
+                "device": "auto",
+                "loadbits": 16,
+                "Huggingface": "stabilityai/stable-video-diffusion-img2vid-xt",
+                "config": {
+                }
             }
         },
         "VtoTModel": {
@@ -726,6 +744,68 @@
                 },
                 "Video Chat Model": {}
             },
+            "Code Model": {
+                "3B Model": {
+                    "stable-code-3b": {
+                        "path": "models/llm/stable-code-3b",
+                        "device": "auto",
+                        "maxmemory": 20,
+                        "cputhreads": 4,
+                        "loadbits": 16,
+                        "preset": "default",
+                        "load_type": "causallm",
+                        "Huggingface": "stabilityai/stable-code-3b"
+                    }
+                },
+                "7B Model": {
+                    "CodeLlama-7b-Python-hf": {
+                        "path": "models/llm/CodeLlama-7b-Python-hf",
+                        "device": "auto",
+                        "maxmemory": 24,
+                        "cputhreads": 4,
+                        "loadbits": 16,
+                        "preset": "default",
+                        "load_type": "llama",
+                        "Huggingface": "codellama/CodeLlama-7b-Python-hf"
+                    },
+                    "CodeLlama-7b-Instruct-hf": {
+                        "path": "models/llm/CodeLlama-7b-Instruct-hf",
+                        "device": "auto",
+                        "maxmemory": 24,
+                        "cputhreads": 4,
+                        "loadbits": 16,
+                        "preset": "default",
+                        "load_type": "llama",
+                        "Huggingface": "codellama/CodeLlama-7b-Instruct-hf"
+                    }
+                },
+                "13B Model": {
+                    "CodeLlama-13b-Python-hf": {
+                        "path": "models/llm/CodeLlama-13b-Python-hf",
+                        "device": "auto",
+                        "maxmemory": 24,
+                        "cputhreads": 4,
+                        "loadbits": 16,
+                        "preset": "default",
+                        "load_type": "llama",
+                        "Huggingface": "codellama/CodeLlama-13b-Python-hf"
+                    },
+                    "CodeLlama-13b-Instruct-hf": {
+                        "path": "models/llm/CodeLlama-13b-Instruct-hf",
+                        "device": "auto",
+                        "maxmemory": 24,
+                        "cputhreads": 4,
+                        "loadbits": 16,
+                        "preset": "default",
+                        "load_type": "llama",
+                        "Huggingface": "codellama/CodeLlama-13b-Instruct-hf"
+                    }
+                },
+                "34B Model": {
+                },
+                "70B Model": {
+                }
+            },
             "Special Model": {
                 "3B Model": {
                     "phi-2-gguf": {

WebUI/webui_pages/dialogue/dialogue.py

@@ -432,6 +432,11 @@ def on_feedback(
     prompt = st.chat_input(chat_input_placeholder, key="prompt", disabled=disabled)
     if voice_prompt != "":
         prompt = voice_prompt
+
+    if modelinfo["mtype"] != ModelType.Code:
+        imagesdata = []
+        audiosdata = []
+        videosdata = []
 
     if prompt != None and prompt != "":
         print("prompt: ", prompt)
@@ -455,7 +460,7 @@ def on_feedback(
             chat_box.ai_say(["Thinking...", ""])
             text = ""
             chat_history_id = ""
-            if imagegeneration_model:
+            if imagegeneration_model and modelinfo["mtype"] != ModelType.Code:
                 imageprompt = ""
                 if imagesprompt:
                     imageprompt = imagesprompt[0]
@@ -484,7 +489,7 @@ def on_feedback(
                 "chat_history_id": chat_history_id,
                 }
             chat_box.update_msg(text, element_index=0, streaming=False, metadata=metadata)
-            if imagegeneration_model:
+            if imagegeneration_model and modelinfo["mtype"] != ModelType.Code:
                 with st.spinner(f"Image generation in progress...."):
                     gen_image = api.get_image_generation_data(text)
                     if gen_image:

WebUI/webui_pages/model_configuration/configuration.py

@@ -50,7 +50,7 @@ def configuration_page(api: ApiRequest, is_lite: bool = False):
         current_model["mtype"] = GetModelType(modeltype)
         type_index = current_model["mtype"].value - 1
     with col2:
-        if type_index == ModelType.Local.value - 1 or type_index == ModelType.Special.value - 1:
+        if type_index == ModelType.Local.value - 1 or type_index == ModelType.Special.value - 1 or type_index == ModelType.Code.value - 1:
             modelsize = st.selectbox(
                     "Please Select Model Size",
                     glob_model_size_list,
@@ -177,7 +177,7 @@ def configuration_page(api: ApiRequest, is_lite: bool = False):
     st.divider()
     if current_model["config"]:
         preset_list = GetPresetPromptList()
-        if current_model["mtype"] == ModelType.Local or current_model["mtype"] == ModelType.Multimodal or current_model["mtype"] == ModelType.Special:
+        if current_model["mtype"] == ModelType.Local or current_model["mtype"] == ModelType.Multimodal or current_model["mtype"] == ModelType.Special or current_model["mtype"] == ModelType.Code:
             tabparams, tabquant, tabtunning, tabprompt = st.tabs(["Parameters", "Quantization", "Fine-Tunning", "Prompt Templates"])
             with tabparams:
                 with st.form("Parameter"):
@@ -407,7 +407,6 @@ def configuration_page(api: ApiRequest, is_lite: bool = False):
                         savename = current_model["mname"]
                         current_model["mname"] = onlinemodel
                         with st.spinner(f"Saving online config, Please do not perform any actions or refresh the page."):
-                            print("current_model: ", current_model)
                             r = api.save_model_config(current_model)
                             if msg := check_error_msg(r):
                                 st.toast(msg, icon="✖")

WebUI/webui_pages/utils.py

@@ -187,7 +187,7 @@ def chat_chat(
             response = self.post("/chat/chat", json=data, stream=True, **kwargs)
             return self._httpx_stream2generator(response, as_json=True)
 
-        elif modelinfo["mtype"] == ModelType.Special or modelinfo["mtype"] == ModelType.Multimodal:
+        elif modelinfo["mtype"] == ModelType.Special or modelinfo["mtype"] == ModelType.Code or modelinfo["mtype"] == ModelType.Multimodal:
             response = self.post(
                "/llm_model/chat",
                json=data,

__webgui_server__.py

@@ -24,6 +24,7 @@
 from WebUI.configs.webuiconfig import *
 from WebUI.configs.basicconfig import *
 from WebUI.configs.specialmodels import *
+from WebUI.configs.codemodels import *
 from typing import Union, List, Dict
 
 def parse_args() -> argparse.ArgumentParser:
@@ -206,7 +207,7 @@ def release_worker(
                     models = app._controller.list_models()
                     if model_name not in models:
                         break
-                elif modelinfo["mtype"] == ModelType.Special or modelinfo["mtype"] == ModelType.Online:
+                elif modelinfo["mtype"] == ModelType.Special or modelinfo["mtype"] == ModelType.Code or modelinfo["mtype"] == ModelType.Online:
                     with get_httpx_client() as client:
                         try:
                             r = client.post(worker_address + "/get_name",
@@ -718,6 +719,12 @@ def create_model_worker_app(log_level: str = "INFO", **kwargs) -> Union[FastAPI,
         MakeFastAPIOffline(app)
         app.title = f"Multimodal Model ({args.model_names[0]})"
         return app
+    # Code model
+    elif kwargs.get("code_model", False) == True:
+        init_code_models(app, args)
+        MakeFastAPIOffline(app)
+        app.title = f"Code Model ({args.model_names[0]})"
+        return app
     # Special model
     elif kwargs.get("special_model", False) == True:
         init_special_models(app, args)

readme-cn.md

@@ -85,17 +85,16 @@
 
   4. 图像识别和图像生成功能的演示：
 
-  图生图演示:
+  文生图的演示，将自然语言转换为图像生成模型的输入语言CLIP，来达到使用自然语言生图的目标:
 
   ![Image](./img/image_to_image_1.png) | ![Image](./img/image_to_image_2.png)
 
   ![Image](./img/image_to_image_3.png) | ![Image](./img/image_to_image_4.png)
 
-  根据图片中的物品，创建手工艺品:
+  根据图片中的物品，创建手工艺品。识别输入图片内容加上自然语言的目标描述，并最终转换为CLIP语言，来达到图生图的目标:
 
   ![Image](./img/image_creative.png)
 
-
 ## 项目介绍
 由三个主界面组成，语言模型的聊天界面，语言模型的配置界面，辅助模型的工具和代理界面。
 
@@ -228,9 +227,10 @@
     在配置界面中，可以选择合适的语言模型加载，语言模型分为`基础模型`，`多模态模型`，`特殊模型`和`在线模型`
 
   1. **`基础模型`** Huggingface上发布的未经量化和处理过的模型，并且它们需要支持和OpenAI相同聊天模版的模型
-  2. **`多模态模型`**(`当前功能还未实现`) 在底层支持语音和文本，或者图片和文本的模型
-  3. **`特殊模型`** Huggingface上发布的量化模型(GGUF)或者需要特殊聊天模版的模型
-  4. **`在线模型`** 支持OpenAI和Google的在线语言模型，比如GPT4-Turbo和Gemini-Pro模型，或者在线的多模态模型GPT4-vision和Gemini-Pro-vision。需要提供OpenAI的API Key和Google API Key。可以在系统环境变量中配置OPENAI_API_KEY和GOOGLE_API_KEY，或者在配置界面中单独配置。
+  2. **`多模态模型`** 在底层支持语音和文本，或者图片和文本的模型
+  3. **`代码模型`** 专业的代码生成模型
+  4. **`特殊模型`** Huggingface上发布的量化模型(GGUF)或者需要特殊聊天模版的模型
+  5. **`在线模型`** 支持OpenAI和Google的在线语言模型，比如GPT4-Turbo和Gemini-Pro模型，或者在线的多模态模型GPT4-vision和Gemini-Pro-vision。需要提供OpenAI的API Key和Google API Key。可以在系统环境变量中配置OPENAI_API_KEY和GOOGLE_API_KEY，或者在配置界面中单独配置。
 
   `目前参数可以配置加载设备和加载位数，模型量化，模型微调，和提示词模版配置功能还未实现`
 
@@ -245,8 +245,8 @@
   2. **`代码执行模型`** (`当前功能还未实现`)
   3. **`文本转语音模型`** 支持本地模型XTTS-v2，支持Azure在线文本转语音服务，需要提供Azure的API Key。也可以在系统环境变量中配置SPEECH_KEY和SPEECH_REGION，或者在配置界面中单独配置。
   4. **`语音转文本模型`** 支持本地模型whisper，fast-whisper，支持Azure在线语音转文本服务，需要提供Azure的API Key。也可以在系统环境变量中配置SPEECH_KEY和SPEECH_REGION，或者在配置界面中单独配置。
-  5. **`图像识别模型`** (`当前功能还未实现`)
-  6. **`图像生成模型`** (`当前功能还未实现`)
+  5. **`图像识别模型`** 支持本地模型blip-image-captioning-large。
+  6. **`图像生成模型`** 支持本地模型OpenDalleV1.1进行静态图片的生成，支持本地模型stable-video-diffusion-img2vid-xt进行动态图片的生成。
   7. **`功能调用`** (`当前功能还未实现`)
 
 ![Image](./img/Tools_Agent.png)
@@ -443,7 +443,24 @@
 
   4. **`图像识别模型`**
 
-      给语言模型提供图像和视频的输入和输出功能，为大脑加上眼睛和绘画能力。`该功能还未实现`
+      给语言模型提供图像和视频的输入和输出功能，为大脑加上眼睛和绘画能力。
+
+      目前支持的图片类型：
+
+      png, jpg, jpeg, bmp
+
+      图像识别的演示：
+      ![Image1](./img/image_creative.png)
+
+      静态图像生成的演示：
+      ![Image1](./img/text_to_image_1.png)
+      ![Image1](./img/text_to_image_2.png)
+      ![Image1](./img/text_to_image_3.png)
+      ![Image1](./img/text_to_image_4.png)
+      根据自然语言文字的描述，生成对应的图片。
+
+      动态图像生成的演示:
+      ![Image1](./img/dynamic_image_1.gif)
 
   5. **`函数定义`**