76 lines
2.1 KiB
Python
76 lines
2.1 KiB
Python
from langdetect import detect
|
|
from google_trans_new import google_translator
|
|
import pandas as pd
|
|
import multiprocessing as mp
|
|
|
|
from database import Data_table
|
|
from database import insert_translated
|
|
|
|
def translate(text, target_lang='en'):
|
|
try:
|
|
input_lang = detect(text)
|
|
except Exception as e:
|
|
print(f'error when detecting language: {e}')
|
|
return ''
|
|
if input_lang == target_lang: return text
|
|
translator = google_translator()
|
|
try:
|
|
text_trans = translator.translate(text, lang_src=input_lang, lang_tgt=target_lang)
|
|
except Exception as e:
|
|
print(f'error when translating text: {e}')
|
|
return ''
|
|
return text_trans
|
|
|
|
def get_untranslated():
|
|
with Data_table() as dt:
|
|
query = (
|
|
f'SELECT * FROM {dt._name} '
|
|
'WHERE content_en IS NULL '
|
|
'ORDER BY random() '
|
|
'LIMIT 1 '
|
|
)
|
|
df = pd.read_sql_query(query, dt.con)
|
|
if len(df)==0: return None
|
|
val_dict = {
|
|
'id': df['id'][0],
|
|
'content': df['content'][0]
|
|
}
|
|
return val_dict
|
|
|
|
def process_translate(val_dict):
|
|
# translate
|
|
try:
|
|
content_en = translate(val_dict['content'])
|
|
except Exception as e:
|
|
content_en = ''
|
|
val_dict['content_en'] = content_en
|
|
# upload to database
|
|
insert_translated(val_dict)
|
|
|
|
def yield_untranslated():
|
|
val_dict = get_untranslated()
|
|
while val_dict is not None:
|
|
yield val_dict
|
|
val_dict = get_untranslated()
|
|
|
|
# def continued_translation():
|
|
# while True:
|
|
# # get untranslated
|
|
# val_dict = get_untranslated()
|
|
# if val_dict is None: break
|
|
# # translate
|
|
# try:
|
|
# content_en = translate(val_dict['content'])
|
|
# except Exception as e:
|
|
# content_en = ''
|
|
# val_dict['content_en'] = content_en
|
|
# # upload to database
|
|
# insert_translated(val_dict)
|
|
|
|
def translate_all_fast():
|
|
with mp.get_context('spawn').Pool(mp.cpu_count()-1) as p:
|
|
list(p.map(process_translate, yield_untranslated()))
|
|
|
|
if __name__ == '__main__':
|
|
translate_all_fast()
|