Skip to content

Commit

Permalink
[fix] jieba temp dir
Browse files Browse the repository at this point in the history
  • Loading branch information
bojieli committed Sep 9, 2024
1 parent 8e159ac commit e0e4651
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 4 deletions.
3 changes: 3 additions & 0 deletions app/models/searchcache.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# This module has two classes: CourseSearchCache and ReviewSearchCache
# Data inside shall be preprocessed with jieba, and stored in the database.
# For now as we have jieba, we don't use ngram.
import os
from app import db, app
from .course import Course
from .review import Review
Expand All @@ -9,6 +10,8 @@


auto_update = app.config.get("UPDATE_SEARCH_CACHE", False)
jieba.dt.tmp_dir = os.path.expanduser("~/.cache/jieba")
os.makedirs(jieba.dt.tmp_dir, exist_ok=True)


# To handle these queries:
Expand Down
4 changes: 1 addition & 3 deletions app/views/ai/prompt_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,11 @@ def generate_summary_prompt(reviews, expected_summary_length):
contents.append(content)

joined_contents = '\n\n'.join(contents)
if len(joined_contents) <= SUMMARY_EXPECTED_LENGTH:
return joined_contents

full_prompt = header + joined_contents
if len(full_prompt) <= PROMPT_LENGTH_LIMIT:
user_prompt = full_prompt
else:
user_prompt = header + generate_short_prompt(reviews, full_prompt)

system_prompt = '你是 USTC 评课社区的一个课程总结助手,旨在为每门课程的点评生成简洁、客观、全面的总结。'
return system_prompt, user_prompt
5 changes: 4 additions & 1 deletion app/views/search/sqlcache.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,21 @@
from typing import List
import os
import re
from app.models import Course, CourseSearchCache, CourseRate, Review, ReviewSearchCache, CourseTerm, ReviewComment
from app.models.searchcache import is_chinese_stop_char
from flask_sqlalchemy.pagination import Pagination
from sqlalchemy import or_
from sqlalchemy.orm import lazyload, load_only
import jieba
import re
# from app.utils import print_sqlalchemy_statement


filter = lambda x: re.sub(r"""[~`!@#$%^&*{}\[\]\\:\";'<>,/\+\-\~\(\)><,、。:【】()?“”「」·\x00-\x1F\x7F]""", " ", x)


def init() -> None:
jieba.dt.tmp_dir = os.path.expanduser("~/.cache/jieba")
os.makedirs(jieba.dt.tmp_dir, exist_ok=True)
jieba.initialize()


Expand Down

0 comments on commit e0e4651

Please sign in to comment.