diff --git a/code/translate.ipynb b/code/translate.ipynb new file mode 100644 index 0000000..5cf543d --- /dev/null +++ b/code/translate.ipynb @@ -0,0 +1,164 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': 108482,\n", + " 'content': 'Chat, flirt, love!\\ntilmeld dig GRATIS nu:\\n\\nhttps://t.co/5Vk2bxjdqm\\n\\n#Danmark #Århus #Odense #Copenhagen #Denmark https://t.co/3XbIv2jImj'}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from translate import get_untranslated\n", + "\n", + "get_untranslated()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'id': 151028, 'content': 'Åbner i vinterferien: Nyt børnemuseum holder åbningsfest med Sigurd Barrett Odense: Har man nysgerrige børn med sa… https://t.co/wsy3JBodsw'}\n", + "{'id': 89329, 'content': 'Guitarkabinet, Marshall Jcm 900, 300 W, Lækkert..., 1.500 kr.,- i Odense V 5200 - https://t.co/UY0x8rTnKH https://t.co/aCOd9CmE80'}\n", + "{'id': 123474, 'content': 'ODEON søger kreativ bager https://t.co/NznIkPH6qc Job i Odense'}\n", + "{'id': 121074, 'content': 'Kommunikationsansvarlig for UngEnergi Odense (UngEnergi) https://t.co/wMnSJ9bifG Job i Odense'}\n", + "{'id': 49585, 'content': '37 Year Ago\\n#LiverpoolFC #LFC #TheReds #Liverpool #BKOdense #BKO #TheStriped #Odense https://t.co/0wJxft6RCD'}\n", + "{'id': 112745, 'content': 'Regnskabschef med IT-ansvar https://t.co/2orZw0VdQA Job i Odense'}\n", + "{'id': 110761, 'content': 'Bugün 2-2 / 3-1 bitebilecek maçlar:\\n\\n1. Leipzig-Köln\\n2-2 (14.60)\\n3-1 (10.60)\\n\\n2. Man City 3-1 Arsenal\\n3. Lyon 3-1St.Etienne (10.20)\\n\\n4. Kopenhag-Odense\\n2-2 (12.50)\\n3-1 (9.25) ⭐️\\n\\n5. Palace-Tottenham\\n2-2 (16.00)\\n1-3 (10.20) ⭐️'}\n", + "{'id': 130457, 'content': 'LÆKKER TAPAS MED LÆKKER DAME #myntejensen ❤️ @ Odense, Denmark https://t.co/TwCNXwmDLy'}\n", + "{'id': 53302, 'content': 'Elguitar, Jackson SL3X soloist, Sælger min..., 2.000 kr.,- i Odense C 5000 - https://t.co/P9FPoAMA5L https://t.co/WJ3Jolnj8j'}\n", + "{'id': 103309, 'content': 'Spillemyndigheden søger #akademisk medarbejder i #Odense. Læs mere på https://t.co/4Ac7RVXazo #jobdk #jobiOdense'}\n", + "{'id': 135454, 'content': 'lvl 4 Blastoise raid until 21:31:51 (58m 32s) near Østerbæksvej 83, Odense M. https://t.co/4qRn0maKUu'}\n", + "{'id': 146330, 'content': 'Guitarcombo, Two rock Coral, 22 W, Med celestion..., kr. 15.500,- i Odense C 5000 - https://t.co/ATCr7nVXXh https://t.co/Qiia2bDxlh'}\n", + "{'id': 115928, 'content': 'Demens i Odense Demens eller Alzheimers sygdom er en folkesygdom og den femte mest hyppige dødsårsag i Danmark.\\nI takt med at antallet og andelen af ældre borgere stiger i Danmark og i Odense Kommune, stiger også ant… https://t.co/S0wFjnWN5c'}\n", + "{'id': 138345, 'content': 'Danmarks størreste #sex #dating portal:\\n\\nhttps://t.co/AnhrlR5efT\\n\\n#Danmark #Denmark\\n#Roskilde #Odense #Copenhagen https://t.co/z39Jz41XGW'}\n", + "{'id': 88562, 'content': 'Job opening: Director, Centre on Population Dynamics (CPOP), University of Southern Denmark, Odense https://t.co/hMW0WMMgMa Deadline for applications: 20 December 2018'}\n", + "{'id': 118162, 'content': 'Dabbing 🙅🏼\\u200d♂️ em Odense, Denmark https://t.co/J5RDVhLD7Z'}\n", + "{'id': 60766, 'content': 'Odense Kommune søger #teamleder, sundhed i #Odense. Læs mere på https://t.co/WU9UTz3LnF #jobdk #jobiOdense'}\n", + "{'id': 149516, 'content': 'Danmarks størreste #sex #dating portal:\\n\\nhttps://t.co/AnhrlR5efT\\n\\n#Danmark #Denmark\\n#Roskilde #Odense #Copenhagen https://t.co/6IR7VqoeUp'}\n", + "{'id': 134039, 'content': '28-årig fra Odense dømt for drugrape af to kvinder https://t.co/XrJgnVTzt4'}\n", + "{'id': 63395, 'content': 'Inspiring and interesting talk\\n #ecgbl2019 @odense about #seriousgames for girls @serenagame #diversity #gbl #gamedev @p_spangen https://t.co/EX7wYoQmBy'}\n", + "{'id': 123129, 'content': 'Klar i nyhedsstudiet med dagens fynske nyheder. God efterårsferie derude 😀 #tv2fyn #odense #tv #nyheder https://t.co/0cUPo15Q4n'}\n", + "{'id': 78825, 'content': 'Today all the houses in my street had smoke detectors installed. #bettersafethansorry #smokedetector #fire #safetyfirst #Odense #Denmark @ Odense, Denmark https://t.co/s8D5Ikd4D5'}\n", + "{'id': 18288, 'content': 'Highlights: SønderjyskE – Odense Bulldogs https://t.co/W0fPazU1Pr #Highlights #hockeymagasinetdk'}\n", + "{'id': 115120, 'content': 'Seniorskolen i Nørre Aaby klar til ny sæson Nr. Aaby: Seniorskolen, som holder til i Nr. Aaby Sognehus og mødes udvalgte onsdag klokken 14, er klar til en ny sæson.\\n10. januar handler det om strejftog i Højskolesangbogen med leder af Odense Mot… … https://t.co/VSU5Je8Y4x'}\n", + "{'id': 125121, 'content': 'Odense Kommune søger #områdechef i #Odense. Læs mere på https://t.co/pe9H1SsO3Q #jobdk #jobiOdense'}\n", + "{'id': 84382, 'content': 'Someone from Odense, Denmark 🇩🇰 ordered this Military Watch Quartz Leather.\\n\\n 🛒 You too, you can get yours at ➡ https://t.co/JIt9HqnezT . https://t.co/8oHRnkw5Lm'}\n", + "{'id': 71585, 'content': '#zulucs Er først lige kommet på. Har I et par kommentarer til det såkaldte DM i esport i Odense i dag? Er det fedt for CS med DM? Hvad med forholdet mellem lange onlineturneringer og korte LANs? Er der overhovedet noget fest i et CS-DM? Og tillykke til Aab #hfesport'}\n", + "{'id': 125496, 'content': 'Soccer #Livescore: (DEN-3R) #FCAmager vs #Odense: 1-1. 2nd Half Started ...'}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Traceback (most recent call last):\n", + " File \"/home/brian/projects/twitter_scraper/code/translate.py\", line 32, in get_untranslated\n", + " df = pd.read_sql_query(query, dt.con)\n", + " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py\", line 397, in read_sql_query\n", + " return pandas_sql.read_query(\n", + " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py\", line 1560, in read_query\n", + " result = self.execute(*args)\n", + " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py\", line 1405, in execute\n", + " return self.connectable.execution_options().execute(*args, **kwargs)\n", + " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 1365, in execute\n", + " return self._exec_driver_sql(\n", + " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 1669, in _exec_driver_sql\n", + " ret = self._execute_context(\n", + " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 1943, in _execute_context\n", + " self._handle_dbapi_exception(\n", + " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 2128, in _handle_dbapi_exception\n", + " util.raise_(exc_info[1], with_traceback=exc_info[2])\n", + " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py\", line 211, in raise_\n", + " raise exception\n", + " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 1900, in _execute_context\n", + " self.dialect.do_execute(\n", + " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py\", line 736, in do_execute\n", + " cursor.execute(statement, parameters)\n", + " File \"/usr/lib/python3.8/encodings/utf_8.py\", line 15, in decode\n", + " def decode(input, errors='strict'):\n", + "KeyboardInterrupt\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[2], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mtranslate\u001b[39;00m \u001b[39mimport\u001b[39;00m yield_untranslated\n\u001b[0;32m----> 3\u001b[0m \u001b[39mfor\u001b[39;00m val_dict \u001b[39min\u001b[39;00m yield_untranslated():\n\u001b[1;32m 4\u001b[0m \u001b[39mprint\u001b[39m(val_dict)\n", + "File \u001b[0;32m~/projects/twitter_scraper/code/translate.py:54\u001b[0m, in \u001b[0;36myield_untranslated\u001b[0;34m()\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[39mwhile\u001b[39;00m val_dict \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 53\u001b[0m \u001b[39myield\u001b[39;00m val_dict\n\u001b[0;32m---> 54\u001b[0m val_dict \u001b[39m=\u001b[39m get_untranslated()\n", + "File \u001b[0;32m~/projects/twitter_scraper/code/translate.py:32\u001b[0m, in \u001b[0;36mget_untranslated\u001b[0;34m()\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[39mwith\u001b[39;00m Data_table() \u001b[39mas\u001b[39;00m dt:\n\u001b[1;32m 26\u001b[0m query \u001b[39m=\u001b[39m (\n\u001b[1;32m 27\u001b[0m \u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39mSELECT * FROM \u001b[39m\u001b[39m{\u001b[39;00mdt\u001b[39m.\u001b[39m_name\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 28\u001b[0m \u001b[39m'\u001b[39m\u001b[39mWHERE content_en IS NULL \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 29\u001b[0m \u001b[39m'\u001b[39m\u001b[39mORDER BY random() \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 30\u001b[0m \u001b[39m'\u001b[39m\u001b[39mLIMIT 1 \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 31\u001b[0m )\n\u001b[0;32m---> 32\u001b[0m df \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39;49mread_sql_query(query, dt\u001b[39m.\u001b[39;49mcon)\n\u001b[1;32m 33\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(df)\u001b[39m==\u001b[39m\u001b[39m0\u001b[39m: \u001b[39mreturn\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 34\u001b[0m val_dict \u001b[39m=\u001b[39m {\n\u001b[1;32m 35\u001b[0m \u001b[39m'\u001b[39m\u001b[39mid\u001b[39m\u001b[39m'\u001b[39m: df[\u001b[39m'\u001b[39m\u001b[39mid\u001b[39m\u001b[39m'\u001b[39m][\u001b[39m0\u001b[39m],\n\u001b[1;32m 36\u001b[0m \u001b[39m'\u001b[39m\u001b[39mcontent\u001b[39m\u001b[39m'\u001b[39m: df[\u001b[39m'\u001b[39m\u001b[39mcontent\u001b[39m\u001b[39m'\u001b[39m][\u001b[39m0\u001b[39m]\n\u001b[1;32m 37\u001b[0m }\n", + "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py:397\u001b[0m, in \u001b[0;36mread_sql_query\u001b[0;34m(sql, con, index_col, coerce_float, params, parse_dates, chunksize, dtype)\u001b[0m\n\u001b[1;32m 339\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 340\u001b[0m \u001b[39mRead SQL query into a DataFrame.\u001b[39;00m\n\u001b[1;32m 341\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 394\u001b[0m \u001b[39mparameter will be converted to UTC.\u001b[39;00m\n\u001b[1;32m 395\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 396\u001b[0m pandas_sql \u001b[39m=\u001b[39m pandasSQL_builder(con)\n\u001b[0;32m--> 397\u001b[0m \u001b[39mreturn\u001b[39;00m pandas_sql\u001b[39m.\u001b[39;49mread_query(\n\u001b[1;32m 398\u001b[0m sql,\n\u001b[1;32m 399\u001b[0m index_col\u001b[39m=\u001b[39;49mindex_col,\n\u001b[1;32m 400\u001b[0m params\u001b[39m=\u001b[39;49mparams,\n\u001b[1;32m 401\u001b[0m coerce_float\u001b[39m=\u001b[39;49mcoerce_float,\n\u001b[1;32m 402\u001b[0m parse_dates\u001b[39m=\u001b[39;49mparse_dates,\n\u001b[1;32m 403\u001b[0m chunksize\u001b[39m=\u001b[39;49mchunksize,\n\u001b[1;32m 404\u001b[0m dtype\u001b[39m=\u001b[39;49mdtype,\n\u001b[1;32m 405\u001b[0m )\n", + "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py:1560\u001b[0m, in \u001b[0;36mSQLDatabase.read_query\u001b[0;34m(self, sql, index_col, coerce_float, parse_dates, params, chunksize, dtype)\u001b[0m\n\u001b[1;32m 1512\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1513\u001b[0m \u001b[39mRead SQL query into a DataFrame.\u001b[39;00m\n\u001b[1;32m 1514\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1556\u001b[0m \n\u001b[1;32m 1557\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1558\u001b[0m args \u001b[39m=\u001b[39m _convert_params(sql, params)\n\u001b[0;32m-> 1560\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mexecute(\u001b[39m*\u001b[39;49margs)\n\u001b[1;32m 1561\u001b[0m columns \u001b[39m=\u001b[39m result\u001b[39m.\u001b[39mkeys()\n\u001b[1;32m 1563\u001b[0m \u001b[39mif\u001b[39;00m chunksize \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n", + "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py:1405\u001b[0m, in \u001b[0;36mSQLDatabase.execute\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1403\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mexecute\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m 1404\u001b[0m \u001b[39m\"\"\"Simple passthrough to SQLAlchemy connectable\"\"\"\u001b[39;00m\n\u001b[0;32m-> 1405\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconnectable\u001b[39m.\u001b[39;49mexecution_options()\u001b[39m.\u001b[39;49mexecute(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n", + "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1365\u001b[0m, in \u001b[0;36mConnection.execute\u001b[0;34m(self, statement, *multiparams, **params)\u001b[0m\n\u001b[1;32m 1356\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(statement, util\u001b[39m.\u001b[39mstring_types):\n\u001b[1;32m 1357\u001b[0m util\u001b[39m.\u001b[39mwarn_deprecated_20(\n\u001b[1;32m 1358\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mPassing a string to Connection.execute() is \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 1359\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mdeprecated and will be removed in version 2.0. Use the \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1362\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mdriver-level SQL string.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 1363\u001b[0m )\n\u001b[0;32m-> 1365\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_exec_driver_sql(\n\u001b[1;32m 1366\u001b[0m statement,\n\u001b[1;32m 1367\u001b[0m multiparams,\n\u001b[1;32m 1368\u001b[0m params,\n\u001b[1;32m 1369\u001b[0m _EMPTY_EXECUTION_OPTS,\n\u001b[1;32m 1370\u001b[0m future\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 1371\u001b[0m )\n\u001b[1;32m 1373\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 1374\u001b[0m meth \u001b[39m=\u001b[39m statement\u001b[39m.\u001b[39m_execute_on_connection\n", + "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1669\u001b[0m, in \u001b[0;36mConnection._exec_driver_sql\u001b[0;34m(self, statement, multiparams, params, execution_options, future)\u001b[0m\n\u001b[1;32m 1659\u001b[0m (\n\u001b[1;32m 1660\u001b[0m statement,\n\u001b[1;32m 1661\u001b[0m distilled_params,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1665\u001b[0m statement, distilled_parameters, execution_options\n\u001b[1;32m 1666\u001b[0m )\n\u001b[1;32m 1668\u001b[0m dialect \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdialect\n\u001b[0;32m-> 1669\u001b[0m ret \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_execute_context(\n\u001b[1;32m 1670\u001b[0m dialect,\n\u001b[1;32m 1671\u001b[0m dialect\u001b[39m.\u001b[39;49mexecution_ctx_cls\u001b[39m.\u001b[39;49m_init_statement,\n\u001b[1;32m 1672\u001b[0m statement,\n\u001b[1;32m 1673\u001b[0m distilled_parameters,\n\u001b[1;32m 1674\u001b[0m execution_options,\n\u001b[1;32m 1675\u001b[0m statement,\n\u001b[1;32m 1676\u001b[0m distilled_parameters,\n\u001b[1;32m 1677\u001b[0m )\n\u001b[1;32m 1679\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m future:\n\u001b[1;32m 1680\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_has_events \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mengine\u001b[39m.\u001b[39m_has_events:\n", + "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1943\u001b[0m, in \u001b[0;36mConnection._execute_context\u001b[0;34m(self, dialect, constructor, statement, parameters, execution_options, *args, **kw)\u001b[0m\n\u001b[1;32m 1940\u001b[0m branched\u001b[39m.\u001b[39mclose()\n\u001b[1;32m 1942\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m-> 1943\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_handle_dbapi_exception(\n\u001b[1;32m 1944\u001b[0m e, statement, parameters, cursor, context\n\u001b[1;32m 1945\u001b[0m )\n\u001b[1;32m 1947\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", + "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:2128\u001b[0m, in \u001b[0;36mConnection._handle_dbapi_exception\u001b[0;34m(self, e, statement, parameters, cursor, context)\u001b[0m\n\u001b[1;32m 2124\u001b[0m util\u001b[39m.\u001b[39mraise_(\n\u001b[1;32m 2125\u001b[0m sqlalchemy_exception, with_traceback\u001b[39m=\u001b[39mexc_info[\u001b[39m2\u001b[39m], from_\u001b[39m=\u001b[39me\n\u001b[1;32m 2126\u001b[0m )\n\u001b[1;32m 2127\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 2128\u001b[0m util\u001b[39m.\u001b[39;49mraise_(exc_info[\u001b[39m1\u001b[39;49m], with_traceback\u001b[39m=\u001b[39;49mexc_info[\u001b[39m2\u001b[39;49m])\n\u001b[1;32m 2130\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 2131\u001b[0m \u001b[39mdel\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_reentrant_error\n", + "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py:211\u001b[0m, in \u001b[0;36mraise_\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m 208\u001b[0m exception\u001b[39m.\u001b[39m__cause__ \u001b[39m=\u001b[39m replace_context\n\u001b[1;32m 210\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 211\u001b[0m \u001b[39mraise\u001b[39;00m exception\n\u001b[1;32m 212\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 213\u001b[0m \u001b[39m# credit to\u001b[39;00m\n\u001b[1;32m 214\u001b[0m \u001b[39m# https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/\u001b[39;00m\n\u001b[1;32m 215\u001b[0m \u001b[39m# as the __traceback__ object creates a cycle\u001b[39;00m\n\u001b[1;32m 216\u001b[0m \u001b[39mdel\u001b[39;00m exception, replace_context, from_, with_traceback\n", + "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1900\u001b[0m, in \u001b[0;36mConnection._execute_context\u001b[0;34m(self, dialect, constructor, statement, parameters, execution_options, *args, **kw)\u001b[0m\n\u001b[1;32m 1898\u001b[0m \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m 1899\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m evt_handled:\n\u001b[0;32m-> 1900\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdialect\u001b[39m.\u001b[39;49mdo_execute(\n\u001b[1;32m 1901\u001b[0m cursor, statement, parameters, context\n\u001b[1;32m 1902\u001b[0m )\n\u001b[1;32m 1904\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_has_events \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mengine\u001b[39m.\u001b[39m_has_events:\n\u001b[1;32m 1905\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdispatch\u001b[39m.\u001b[39mafter_cursor_execute(\n\u001b[1;32m 1906\u001b[0m \u001b[39mself\u001b[39m,\n\u001b[1;32m 1907\u001b[0m cursor,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1911\u001b[0m context\u001b[39m.\u001b[39mexecutemany,\n\u001b[1;32m 1912\u001b[0m )\n", + "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py:736\u001b[0m, in \u001b[0;36mDefaultDialect.do_execute\u001b[0;34m(self, cursor, statement, parameters, context)\u001b[0m\n\u001b[1;32m 735\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdo_execute\u001b[39m(\u001b[39mself\u001b[39m, cursor, statement, parameters, context\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m):\n\u001b[0;32m--> 736\u001b[0m cursor\u001b[39m.\u001b[39;49mexecute(statement, parameters)\n", + "File \u001b[0;32m/usr/lib/python3.8/encodings/utf_8.py:15\u001b[0m, in \u001b[0;36mdecode\u001b[0;34m(input, errors)\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[39m### Codec APIs\u001b[39;00m\n\u001b[1;32m 13\u001b[0m encode \u001b[39m=\u001b[39m codecs\u001b[39m.\u001b[39mutf_8_encode\n\u001b[0;32m---> 15\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdecode\u001b[39m(\u001b[39minput\u001b[39m, errors\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mstrict\u001b[39m\u001b[39m'\u001b[39m):\n\u001b[1;32m 16\u001b[0m \u001b[39mreturn\u001b[39;00m codecs\u001b[39m.\u001b[39mutf_8_decode(\u001b[39minput\u001b[39m, errors, \u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m 18\u001b[0m \u001b[39mclass\u001b[39;00m \u001b[39mIncrementalEncoder\u001b[39;00m(codecs\u001b[39m.\u001b[39mIncrementalEncoder):\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "from translate import yield_untranslated\n", + "\n", + "for val_dict in yield_untranslated():\n", + " print(val_dict)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "57b41ea9bda188de64800ef31c5cfb4fcfca577257038becb66e4fa0a5ee97a6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/code/translate.py b/code/translate.py index f28aebd..2cdc3c5 100644 --- a/code/translate.py +++ b/code/translate.py @@ -1,5 +1,10 @@ from langdetect import detect from google_trans_new import google_translator +import pandas as pd +import multiprocessing as mp + +from database import Data_table +from database import insert_translated def translate(text, target_lang='en'): try: @@ -15,3 +20,56 @@ def translate(text, target_lang='en'): print(f'error when translating text: {e}') return '' return text_trans + +def get_untranslated(): + with Data_table() as dt: + query = ( + f'SELECT * FROM {dt._name} ' + 'WHERE content_en IS NULL ' + 'ORDER BY random() ' + 'LIMIT 1 ' + ) + df = pd.read_sql_query(query, dt.con) + if len(df)==0: return None + val_dict = { + 'id': df['id'][0], + 'content': df['content'][0] + } + return val_dict + +def process_translate(val_dict): + # translate + try: + content_en = translate(val_dict['content']) + except Exception as e: + content_en = '' + val_dict['content_en'] = content_en + # upload to database + insert_translated(val_dict) + +def yield_untranslated(): + val_dict = get_untranslated() + while val_dict is not None: + yield val_dict + val_dict = get_untranslated() + +# def continued_translation(): +# while True: +# # get untranslated +# val_dict = get_untranslated() +# if val_dict is None: break +# # translate +# try: +# content_en = translate(val_dict['content']) +# except Exception as e: +# content_en = '' +# val_dict['content_en'] = content_en +# # upload to database +# insert_translated(val_dict) + +def translate_all_fast(): + with mp.get_context('spawn').Pool(mp.cpu_count()-1) as p: + list(p.map(process_translate, yield_untranslated())) + +if __name__ == '__main__': + translate_all_fast()