71 lines
2.1 KiB
Python
71 lines
2.1 KiB
Python
from langdetect import detect
|
|
from google_trans_new import google_translator
|
|
import pandas as pd
|
|
import multiprocessing as mp
|
|
import numpy as np
|
|
from dotenv import load_dotenv
|
|
from tqdm import tqdm
|
|
|
|
from database import Data_table
|
|
from database import insert_translated
|
|
from clean_data import clean_content
|
|
|
|
def translate(text, target_lang='en'):
|
|
try:
|
|
input_lang = detect(text)
|
|
except Exception as e:
|
|
print(f'error when detecting language: {e}')
|
|
return ''
|
|
if input_lang == target_lang: return text
|
|
translator = google_translator()
|
|
try:
|
|
text_trans = translator.translate(text, lang_src=input_lang, lang_tgt=target_lang)
|
|
except Exception as e:
|
|
print(f'error when translating text: {e}')
|
|
return ''
|
|
return text_trans
|
|
|
|
def get_untranslated():
|
|
with Data_table() as dt:
|
|
query = (
|
|
f'SELECT * FROM {dt._name} '
|
|
'WHERE content_en IS NULL '
|
|
'ORDER BY random() '
|
|
'LIMIT 1 '
|
|
)
|
|
df = pd.read_sql_query(query, dt.con)
|
|
if len(df)==0: return None
|
|
val_dict = {
|
|
'id': int(df['id'][0]),
|
|
'content': str(df['content'][0])
|
|
}
|
|
return val_dict
|
|
|
|
def continously_translate():
|
|
with Data_table() as dt:
|
|
query = (
|
|
f'UPDATE {dt._name} '
|
|
' SET content_en = (%(content_en)s) '
|
|
'WHERE id = (%(id)s) '
|
|
)
|
|
while True:
|
|
val_dict = get_untranslated()
|
|
if val_dict is None: break
|
|
# clean
|
|
content_clean = clean_content(val_dict['content'])
|
|
# translate
|
|
try:
|
|
content_en = translate(content_clean)
|
|
except Exception as e:
|
|
content_en = ''
|
|
val_dict['content_en'] = content_en
|
|
# upload to database
|
|
dt.execute(query, val_dict)
|
|
print('translated content')
|
|
print('no more untranslated content located')
|
|
|
|
if __name__ == '__main__':
|
|
load_dotenv()
|
|
continously_translate()
|
|
# later try to set all empty strings back to null and retranslate failed...
|