|
from toolbox import update_ui |
|
from toolbox import CatchException, report_execption, write_results_to_file |
|
fast_debug = False |
|
|
|
class PaperFileGroup(): |
|
def __init__(self): |
|
self.file_paths = [] |
|
self.file_contents = [] |
|
self.sp_file_contents = [] |
|
self.sp_file_index = [] |
|
self.sp_file_tag = [] |
|
|
|
|
|
from request_llm.bridge_all import model_info |
|
enc = model_info["gpt-3.5-turbo"]['tokenizer'] |
|
def get_token_num(txt): return len(enc.encode(txt, disallowed_special=())) |
|
self.get_token_num = get_token_num |
|
|
|
def run_file_split(self, max_token_limit=1900): |
|
""" |
|
将长文本分离开来 |
|
""" |
|
for index, file_content in enumerate(self.file_contents): |
|
if self.get_token_num(file_content) < max_token_limit: |
|
self.sp_file_contents.append(file_content) |
|
self.sp_file_index.append(index) |
|
self.sp_file_tag.append(self.file_paths[index]) |
|
else: |
|
from .crazy_utils import breakdown_txt_to_satisfy_token_limit_for_pdf |
|
segments = breakdown_txt_to_satisfy_token_limit_for_pdf(file_content, self.get_token_num, max_token_limit) |
|
for j, segment in enumerate(segments): |
|
self.sp_file_contents.append(segment) |
|
self.sp_file_index.append(index) |
|
self.sp_file_tag.append(self.file_paths[index] + f".part-{j}.md") |
|
|
|
print('Segmentation: done') |
|
|
|
def 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en'): |
|
import time, os, re |
|
from .crazy_utils import request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency |
|
|
|
|
|
pfg = PaperFileGroup() |
|
|
|
for index, fp in enumerate(file_manifest): |
|
with open(fp, 'r', encoding='utf-8', errors='replace') as f: |
|
file_content = f.read() |
|
|
|
pfg.file_paths.append(fp) |
|
pfg.file_contents.append(file_content) |
|
|
|
|
|
pfg.run_file_split(max_token_limit=1500) |
|
n_split = len(pfg.sp_file_contents) |
|
|
|
|
|
if language == 'en->zh': |
|
inputs_array = ["This is a Markdown file, translate it into Chinese, do not modify any existing Markdown commands:" + |
|
f"\n\n{frag}" for frag in pfg.sp_file_contents] |
|
inputs_show_user_array = [f"翻译 {f}" for f in pfg.sp_file_tag] |
|
sys_prompt_array = ["You are a professional academic paper translator." for _ in range(n_split)] |
|
elif language == 'zh->en': |
|
inputs_array = [f"This is a Markdown file, translate it into English, do not modify any existing Markdown commands:" + |
|
f"\n\n{frag}" for frag in pfg.sp_file_contents] |
|
inputs_show_user_array = [f"翻译 {f}" for f in pfg.sp_file_tag] |
|
sys_prompt_array = ["You are a professional academic paper translator." for _ in range(n_split)] |
|
|
|
gpt_response_collection = yield from request_gpt_model_multi_threads_with_very_awesome_ui_and_high_efficiency( |
|
inputs_array=inputs_array, |
|
inputs_show_user_array=inputs_show_user_array, |
|
llm_kwargs=llm_kwargs, |
|
chatbot=chatbot, |
|
history_array=[[""] for _ in range(n_split)], |
|
sys_prompt_array=sys_prompt_array, |
|
|
|
scroller_max_len = 80 |
|
) |
|
|
|
|
|
create_report_file_name = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + f"-chatgpt.polish.md" |
|
res = write_results_to_file(gpt_response_collection, file_name=create_report_file_name) |
|
history = gpt_response_collection |
|
chatbot.append((f"{fp}完成了吗？", res)) |
|
yield from update_ui(chatbot=chatbot, history=history) |
|
|
|
|
|
def get_files_from_everything(txt): |
|
import glob, os |
|
|
|
success = True |
|
if txt.startswith('http'): |
|
|
|
txt = txt.replace("https://github.com/", "https://raw.githubusercontent.com/") |
|
txt = txt.replace("/blob/", "/") |
|
import requests |
|
from toolbox import get_conf |
|
proxies, = get_conf('proxies') |
|
r = requests.get(txt, proxies=proxies) |
|
with open('./gpt_log/temp.md', 'wb+') as f: f.write(r.content) |
|
project_folder = './gpt_log/' |
|
file_manifest = ['./gpt_log/temp.md'] |
|
elif txt.endswith('.md'): |
|
|
|
file_manifest = [txt] |
|
project_folder = os.path.dirname(txt) |
|
elif os.path.exists(txt): |
|
|
|
project_folder = txt |
|
file_manifest = [f for f in glob.glob(f'{project_folder}/**/*.md', recursive=True)] |
|
else: |
|
success = False |
|
|
|
return success, file_manifest, project_folder |
|
|
|
|
|
@CatchException |
|
def Markdown英译中(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): |
|
|
|
chatbot.append([ |
|
"函数插件功能？", |
|
"对整个Markdown项目进行翻译。函数插件贡献者: Binary-Husky"]) |
|
yield from update_ui(chatbot=chatbot, history=history) |
|
|
|
|
|
try: |
|
import tiktoken |
|
import glob, os |
|
except: |
|
report_execption(chatbot, history, |
|
a=f"解析项目: {txt}", |
|
b=f"导入软件依赖失败。使用该模块需要额外依赖，安装方法```pip install --upgrade tiktoken```。") |
|
yield from update_ui(chatbot=chatbot, history=history) |
|
return |
|
history = [] |
|
|
|
success, file_manifest, project_folder = get_files_from_everything(txt) |
|
|
|
if not success: |
|
|
|
if txt == "": txt = '空空如也的输入栏' |
|
report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") |
|
yield from update_ui(chatbot=chatbot, history=history) |
|
return |
|
|
|
if len(file_manifest) == 0: |
|
report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.md文件: {txt}") |
|
yield from update_ui(chatbot=chatbot, history=history) |
|
return |
|
|
|
yield from 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='en->zh') |
|
|
|
|
|
|
|
|
|
|
|
@CatchException |
|
def Markdown中译英(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port): |
|
|
|
chatbot.append([ |
|
"函数插件功能？", |
|
"对整个Markdown项目进行翻译。函数插件贡献者: Binary-Husky"]) |
|
yield from update_ui(chatbot=chatbot, history=history) |
|
|
|
|
|
try: |
|
import tiktoken |
|
import glob, os |
|
except: |
|
report_execption(chatbot, history, |
|
a=f"解析项目: {txt}", |
|
b=f"导入软件依赖失败。使用该模块需要额外依赖，安装方法```pip install --upgrade tiktoken```。") |
|
yield from update_ui(chatbot=chatbot, history=history) |
|
return |
|
history = [] |
|
success, file_manifest, project_folder = get_files_from_everything(txt) |
|
if not success: |
|
|
|
if txt == "": txt = '空空如也的输入栏' |
|
report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到本地项目或无权访问: {txt}") |
|
yield from update_ui(chatbot=chatbot, history=history) |
|
return |
|
if len(file_manifest) == 0: |
|
report_execption(chatbot, history, a = f"解析项目: {txt}", b = f"找不到任何.md文件: {txt}") |
|
yield from update_ui(chatbot=chatbot, history=history) |
|
return |
|
yield from 多文件翻译(file_manifest, project_folder, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, language='zh->en') |
