chickenflyshigh/onscreen-translator
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added support for Groqq API models. Created QT6 overlay app.

Browse Source
This commit is contained in:
chickenflyshigh
2024-11-04 15:36:50 +11:00
parent 17e7f6526f
commit 499a2c3972
12 changed files with 699 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
helpers/batching.py
+112 -22
View File
@@ -1,37 +1,63 @@
from torch.utils.data import Dataset, DataLoader
from typing import List, Dict
from dotenv import load_dotenv
import os , sys, torch, time
from multiprocessing import Process, Event
import os , sys, torch, time, ast
from werkzeug.exceptions import TooManyRequests
from multiprocessing import Process, Event, Value
load_dotenv()
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
from config import device
from config import device, GEMINI_API_KEY, GROQ_API_KEY
from logging_config import logger
from groq import Groq
import google.generativeai as genai
import asyncio
from functools import wraps
class Gemini():
def __init__(self, name, rate):
self.name = name
class ApiModel():
def __init__(self, model, # model name
rate, # rate of calls per minute
api_key, # api key for the model wrt the site
):
self.model = model
self.rate = rate
self.curr_calls = 0
self.time = 0
self.api_key = api_key
self.curr_calls = Value('i', 0)
self.time = Value('i', 0)
self.process = None
self.stop_event = Event()
self.site = None
self.from_lang = None
self.target_lang = None
self.request = None # request response from API
def __repr__(self):
return f'Model: {self.name}; Rate: {self.rate}; Current_Calls: {self.curr_calls} calls; Time Passed: {self.time} seconds.'
return f'{self.site} Model: {self.model}; Rate: {self.rate}; Current_Calls: {self.curr_calls.value} calls; Time Passed: {self.time.value} seconds.'
def __str__(self):
return self.name
return self.model
def background_task(self):
def set_lang(self, from_lang, target_lang):
self.from_lang = from_lang
self.target_lang = target_lang
### CHECK MINUTELY API RATES. For working with hourly rates and monthly will need to create another file. Also just unlikely those rates will be hit
async def api_rate_check(self):
# Background task to manage the rate of calls to the API
while not self.stop_event.is_set():
time.sleep(5)
self.time += 5
if self.time >= 60:
self.time = 0
self.curr_calls = 0
start_time = time.monotonic()
self.time.value += 5
if self.time.value >= 60:
self.time.value = 0
self.curr_calls.value = 0
elapsed = time.monotonic() - start_time
# Sleep for exactly 5 seconds minus the elapsed time
sleep_time = max(0, 5 - elapsed)
await asyncio.sleep(sleep_time)
def background_task(self):
asyncio.run(self.api_rate_check())
def start(self):
# Start the background task
@@ -49,6 +75,68 @@ class Gemini():
if self.process.is_alive():
self.process.terminate()
def request_func(request):
@wraps(request)
def wrapper(self, text, *args, **kwargs):
if self.curr_calls.value < self.rate:
# try:
response = request(self, text, *args, **kwargs)
self.curr_calls.value += 1
return response
# except Exception as e:
#logger.error(f"Error with model {self.model} from {self.site}. Error: {e}")
else:
logger.error(f"Rate limit reached for this model. Please wait for the rate to reset in {60 - self.time} seconds.")
raise TooManyRequests('Rate limit reached for this model.')
return wrapper
@request_func
def translate(self, request_fn, texts_to_translate):
if len(texts_to_translate) == 0:
return []
prompt = f"Without any additional remarks, and without any code, translate the following items of the Python list from {self.from_lang} into {self.target_lang} and output as a Python list ensuring proper escaping of characters: {texts_to_translate}"
response = request_fn(self, prompt)
return ast.literal_eval(response.strip())
class Groqq(ApiModel):
def __init__(self, model, rate, api_key = GROQ_API_KEY):
super().__init__(model, rate, api_key)
self.site = "Groq"
def request(self, content):
client = Groq()
chat_completion = client.chat.completions.create(
messages=[
{
"role": "user",
"content": content,
}
],
model=self.model
)
return chat_completion.choices[0].message.content
def translate(self, texts_to_translate):
return super().translate(Groqq.request, texts_to_translate)
class Gemini(ApiModel):
def __init__(self, model, rate, api_key = GEMINI_API_KEY):
super().__init__(model, rate, api_key)
self.site = "Gemini"
def request(self, content):
genai.configure(api_key=self.api_key)
safety_settings = {
"HARM_CATEGORY_HARASSMENT": "BLOCK_NONE",
"HARM_CATEGORY_HATE_SPEECH": "BLOCK_NONE",
"HARM_CATEGORY_SEXUALLY_EXPLICIT": "BLOCK_NONE",
"HARM_CATEGORY_DANGEROUS_CONTENT": "BLOCK_NONE"}
response = genai.GenerativeModel(self.model).generate_content(content, safety_settings=safety_settings)
return response.text.strip()
def translate(self, texts_to_translate):
return super().translate(Gemini.request, texts_to_translate)
class TranslationDataset(Dataset):
def __init__(self, texts: List[str], tokenizer, max_length: int = 512):
"""
@@ -174,9 +262,11 @@ def generate_text(
return all_generated_texts
if __name__ == '__main__':
from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer
model = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M", local_files_only=True).to(device)
tokenizer = M2M100Tokenizer.from_pretrained("facebook/m2m100_418M", local_files_only=True)
tokenizer.src_lang = "zh"
texts = ["你好","我"]
print(generate_text(texts,model, tokenizer, forced_bos_token_id=tokenizer.get_lang_id('en')))
GROQ_API_KEY = os.getenv('GROQ_API_KEY')
GEMINI_API_KEY = os.getenv('GEMINI_API_KEY')
groq = Groqq('gemma-7b-it', 15, GROQ_API_KEY)
groq.set_lang('zh','en')
gemini = Gemini('gemini-1.5-pro', 15, GEMINI_API_KEY)
gemini.set_lang('zh','en')
print(gemini.translate(['荷兰咯']))
print(groq.translate(['荷兰咯']))
helpers/translation.py
+27 -54
View File
@@ -1,15 +1,17 @@
from transformers import M2M100Tokenizer, M2M100ForConditionalGeneration, AutoTokenizer, AutoModelForSeq2SeqLM, GPTQConfig, AutoModelForCausalLM
import google.generativeai as genai
import torch, os, sys, ast
import torch, os, sys, ast, json
from utils import standardize_lang
from functools import wraps
from batching import generate_text, Gemini
import random
import batching
from batching import generate_text, Gemini, Groq
from logging_config import logger
from multiprocessing import Process,Event
# root dir
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
from config import LOCAL_FILES_ONLY, available_langs, curr_models, BATCH_SIZE, device, GEMINI_KEY, MAX_INPUT_TOKENS, MAX_OUTPUT_TOKENS, seq_llm_models, api_llm_models, causal_llm_models
from config import LOCAL_FILES_ONLY, available_langs, curr_models, BATCH_SIZE, device, GEMINI_API_KEY, MAX_INPUT_TOKENS, MAX_OUTPUT_TOKENS, seq_llm_models, api_llm_models, causal_llm_models
##############################
# translation decorator
@@ -27,36 +29,30 @@ def translate(translation_func):
###############################
def init_GEMINI(models_and_rates = None):
if not models_and_rates:
## this is default for free tier
models_and_rates = {'gemini-1.5-pro': 2, 'gemini-1.5-flash': 15, 'gemini-1.5-flash-8b': 8, 'gemini-1.0-pro': 15} # order from most pref to least pref
models = [Gemini(name, rate) for name, rate in models_and_rates.items()]
def init_API_LLM(from_lang, target_lang):
from_lang = standardize_lang(from_lang)['translation_model_lang']
target_lang = standardize_lang(target_lang)['translation_model_lang']
with open('api_models.json', 'r') as f:
models_and_rates = json.load(f)
models = []
for class_type, class_models in models_and_rates.items():
cls = getattr(batching, class_type)
instantiated_objects = [ cls(model, rate) for model, rate in class_models.items()]
models.extend(instantiated_objects)
for model in models:
model.start()
genai.configure(api_key=GEMINI_KEY)
model.set_lang(from_lang, target_lang)
return models
def translate_GEMINI(text, models, from_lang, target_lang):
safety_settings = {
"HARM_CATEGORY_HARASSMENT": "BLOCK_NONE",
"HARM_CATEGORY_HATE_SPEECH": "BLOCK_NONE",
"HARM_CATEGORY_SEXUALLY_EXPLICIT": "BLOCK_NONE",
"HARM_CATEGORY_DANGEROUS_CONTENT": "BLOCK_NONE"}
prompt = f"Without any additional remarks, and without any code, translate the following items of the Python list from {from_lang} into {target_lang} and output as a Python list ensuring proper escaping of characters: {text}"
def translate_API_LLM(text, models):
random.shuffle(models)
for model in models:
if model.curr_calls < model.rate:
try:
response = genai.GenerativeModel(model.name).generate_content(prompt,
safety_settings=safety_settings)
model.curr_calls += 1
logger.info(repr(model))
logger.info(f'Model Response: {response.text.strip()}')
return ast.literal_eval(response.text.strip())
except Exception as e:
logger.error(f"Error with model {model.name}. Error: {e}")
logger.error("No models available to translate. Please wait for a model to be available.")
try:
return model.translate(text)
except:
continue
logger.error("All models have failed to translate the text.")
###############################
# Best model by far. Aya-23-8B. Gemma is relatively good. If I get the time to quantize either gemma or aya those will be good to use. llama3.2 is really good as well.
@@ -127,12 +123,6 @@ def init_Seq_LLM(model_type, **kwargs): # model = 'opus' or 'm2m'
else:
raise ValueError(f"Invalid model. Please use {' or '.join(curr_models)}.")
def init_API_LLM(model_type, **kwargs): # model = 'gemma'
if model_type == 'gemini':
return init_GEMINI(**kwargs)
else:
raise ValueError(f"Invalid model type. Please use {' or '.join(api_llm_models)}.")
def init_Causal_LLM(model_type, **kwargs):
pass
###
@@ -155,23 +145,8 @@ def translate_Seq_LLM(text,
### if you want to use any other translation, just define a translate function with input text and output text.
# def translate_api(text):
#@translate
#def translate_Causal_LLM(text, model_type, model)
@translate
def translate_API_LLM(text: list[str],
model_type: str, # 'gemma'
models: list, # list of objects of classes defined in batching.py
from_lang: str, # suggested to use ISO 639-1 codes
target_lang: str # suggested to use ISO 639-1 codes
) -> list[str]:
if model_type == 'gemini':
from_lang = standardize_lang(from_lang)['translation_model_lang']
target_lang = standardize_lang(target_lang)['translation_model_lang']
return translate_GEMINI(text, models, from_lang, target_lang)
else:
raise ValueError(f"Invalid model. Please use {' or '.join(api_llm_models)}.")
#def translate_Causal_LLM(text, model_type, model)
@translate
def translate_Causal_LLM(text: list[str],
@@ -211,7 +186,5 @@ def translate_func(model):
### todo: if cuda is not detected, default to online translation as cpu just won't cut it bro. Parallel process it over multiple websites to make it faster
if __name__ == "__main__":
models = init_GEMINI()
print(translate_API_LLM(['想要借用合成台。', '有什么卖的?', '声音太大会有什么影响吗?', '不怕丢东西吗?', '再见。', '布纳马', '想买什么自己拿。把钱留在旁边就好。', '回顾', '隐藏'], 'gemini', models, from_lang='ch_sim', target_lang='en'))
# model, tokenizer = init_M2M()
# print(translate_Seq_LLM( ['想要借用合成台。', '有什么卖的?', '声音太大会有什么影响吗?', '不怕丢东西吗?', '再见。', '布纳马', '想买什么自己拿。把钱留在旁边就好。', '回顾', '隐藏'], 'm2m', model, tokenizer, from_lang='ch_sim', target_lang='en'))
models = init_API_LLM('ja', 'en')
print(translate_API_LLM(['こんにちは'], models))