From c1aa7d23eb2afa0b67830141ff41381c4c03dcf7 Mon Sep 17 00:00:00 2001 From: brian Date: Sun, 18 Dec 2022 15:03:53 +0000 Subject: [PATCH] updated code to run on multiple hosts at the same time --- code/translate.ipynb | 98 +++++--------------------------------------- code/translate.py | 56 ++++++++++--------------- 2 files changed, 31 insertions(+), 123 deletions(-) diff --git a/code/translate.ipynb b/code/translate.ipynb index 5cf543d..cded5c2 100644 --- a/code/translate.ipynb +++ b/code/translate.ipynb @@ -29,101 +29,23 @@ "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'id': 151028, 'content': 'Åbner i vinterferien: Nyt børnemuseum holder åbningsfest med Sigurd Barrett Odense: Har man nysgerrige børn med sa… https://t.co/wsy3JBodsw'}\n", - "{'id': 89329, 'content': 'Guitarkabinet, Marshall Jcm 900, 300 W, Lækkert..., 1.500 kr.,- i Odense V 5200 - https://t.co/UY0x8rTnKH https://t.co/aCOd9CmE80'}\n", - "{'id': 123474, 'content': 'ODEON søger kreativ bager https://t.co/NznIkPH6qc Job i Odense'}\n", - "{'id': 121074, 'content': 'Kommunikationsansvarlig for UngEnergi Odense (UngEnergi) https://t.co/wMnSJ9bifG Job i Odense'}\n", - "{'id': 49585, 'content': '37 Year Ago\\n#LiverpoolFC #LFC #TheReds #Liverpool #BKOdense #BKO #TheStriped #Odense https://t.co/0wJxft6RCD'}\n", - "{'id': 112745, 'content': 'Regnskabschef med IT-ansvar https://t.co/2orZw0VdQA Job i Odense'}\n", - "{'id': 110761, 'content': 'Bugün 2-2 / 3-1 bitebilecek maçlar:\\n\\n1. Leipzig-Köln\\n2-2 (14.60)\\n3-1 (10.60)\\n\\n2. Man City 3-1 Arsenal\\n3. Lyon 3-1St.Etienne (10.20)\\n\\n4. Kopenhag-Odense\\n2-2 (12.50)\\n3-1 (9.25) ⭐️\\n\\n5. Palace-Tottenham\\n2-2 (16.00)\\n1-3 (10.20) ⭐️'}\n", - "{'id': 130457, 'content': 'LÆKKER TAPAS MED LÆKKER DAME #myntejensen ❤️ @ Odense, Denmark https://t.co/TwCNXwmDLy'}\n", - "{'id': 53302, 'content': 'Elguitar, Jackson SL3X soloist, Sælger min..., 2.000 kr.,- i Odense C 5000 - https://t.co/P9FPoAMA5L https://t.co/WJ3Jolnj8j'}\n", - "{'id': 103309, 'content': 'Spillemyndigheden søger #akademisk medarbejder i #Odense. Læs mere på https://t.co/4Ac7RVXazo #jobdk #jobiOdense'}\n", - "{'id': 135454, 'content': 'lvl 4 Blastoise raid until 21:31:51 (58m 32s) near Østerbæksvej 83, Odense M. https://t.co/4qRn0maKUu'}\n", - "{'id': 146330, 'content': 'Guitarcombo, Two rock Coral, 22 W, Med celestion..., kr. 15.500,- i Odense C 5000 - https://t.co/ATCr7nVXXh https://t.co/Qiia2bDxlh'}\n", - "{'id': 115928, 'content': 'Demens i Odense Demens eller Alzheimers sygdom er en folkesygdom og den femte mest hyppige dødsårsag i Danmark.\\nI takt med at antallet og andelen af ældre borgere stiger i Danmark og i Odense Kommune, stiger også ant… https://t.co/S0wFjnWN5c'}\n", - "{'id': 138345, 'content': 'Danmarks størreste #sex #dating portal:\\n\\nhttps://t.co/AnhrlR5efT\\n\\n#Danmark #Denmark\\n#Roskilde #Odense #Copenhagen https://t.co/z39Jz41XGW'}\n", - "{'id': 88562, 'content': 'Job opening: Director, Centre on Population Dynamics (CPOP), University of Southern Denmark, Odense https://t.co/hMW0WMMgMa Deadline for applications: 20 December 2018'}\n", - "{'id': 118162, 'content': 'Dabbing 🙅🏼\\u200d♂️ em Odense, Denmark https://t.co/J5RDVhLD7Z'}\n", - "{'id': 60766, 'content': 'Odense Kommune søger #teamleder, sundhed i #Odense. Læs mere på https://t.co/WU9UTz3LnF #jobdk #jobiOdense'}\n", - "{'id': 149516, 'content': 'Danmarks størreste #sex #dating portal:\\n\\nhttps://t.co/AnhrlR5efT\\n\\n#Danmark #Denmark\\n#Roskilde #Odense #Copenhagen https://t.co/6IR7VqoeUp'}\n", - "{'id': 134039, 'content': '28-årig fra Odense dømt for drugrape af to kvinder https://t.co/XrJgnVTzt4'}\n", - "{'id': 63395, 'content': 'Inspiring and interesting talk\\n #ecgbl2019 @odense about #seriousgames for girls @serenagame #diversity #gbl #gamedev @p_spangen https://t.co/EX7wYoQmBy'}\n", - "{'id': 123129, 'content': 'Klar i nyhedsstudiet med dagens fynske nyheder. God efterårsferie derude 😀 #tv2fyn #odense #tv #nyheder https://t.co/0cUPo15Q4n'}\n", - "{'id': 78825, 'content': 'Today all the houses in my street had smoke detectors installed. #bettersafethansorry #smokedetector #fire #safetyfirst #Odense #Denmark @ Odense, Denmark https://t.co/s8D5Ikd4D5'}\n", - "{'id': 18288, 'content': 'Highlights: SønderjyskE – Odense Bulldogs https://t.co/W0fPazU1Pr #Highlights #hockeymagasinetdk'}\n", - "{'id': 115120, 'content': 'Seniorskolen i Nørre Aaby klar til ny sæson Nr. Aaby: Seniorskolen, som holder til i Nr. Aaby Sognehus og mødes udvalgte onsdag klokken 14, er klar til en ny sæson.\\n10. januar handler det om strejftog i Højskolesangbogen med leder af Odense Mot… … https://t.co/VSU5Je8Y4x'}\n", - "{'id': 125121, 'content': 'Odense Kommune søger #områdechef i #Odense. Læs mere på https://t.co/pe9H1SsO3Q #jobdk #jobiOdense'}\n", - "{'id': 84382, 'content': 'Someone from Odense, Denmark 🇩🇰 ordered this Military Watch Quartz Leather.\\n\\n 🛒 You too, you can get yours at ➡ https://t.co/JIt9HqnezT . https://t.co/8oHRnkw5Lm'}\n", - "{'id': 71585, 'content': '#zulucs Er først lige kommet på. Har I et par kommentarer til det såkaldte DM i esport i Odense i dag? Er det fedt for CS med DM? Hvad med forholdet mellem lange onlineturneringer og korte LANs? Er der overhovedet noget fest i et CS-DM? Og tillykke til Aab #hfesport'}\n", - "{'id': 125496, 'content': 'Soccer #Livescore: (DEN-3R) #FCAmager vs #Odense: 1-1. 2nd Half Started ...'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Traceback (most recent call last):\n", - " File \"/home/brian/projects/twitter_scraper/code/translate.py\", line 32, in get_untranslated\n", - " df = pd.read_sql_query(query, dt.con)\n", - " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py\", line 397, in read_sql_query\n", - " return pandas_sql.read_query(\n", - " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py\", line 1560, in read_query\n", - " result = self.execute(*args)\n", - " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py\", line 1405, in execute\n", - " return self.connectable.execution_options().execute(*args, **kwargs)\n", - " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 1365, in execute\n", - " return self._exec_driver_sql(\n", - " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 1669, in _exec_driver_sql\n", - " ret = self._execute_context(\n", - " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 1943, in _execute_context\n", - " self._handle_dbapi_exception(\n", - " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 2128, in _handle_dbapi_exception\n", - " util.raise_(exc_info[1], with_traceback=exc_info[2])\n", - " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py\", line 211, in raise_\n", - " raise exception\n", - " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 1900, in _execute_context\n", - " self.dialect.do_execute(\n", - " File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py\", line 736, in do_execute\n", - " cursor.execute(statement, parameters)\n", - " File \"/usr/lib/python3.8/encodings/utf_8.py\", line 15, in decode\n", - " def decode(input, errors='strict'):\n", - "KeyboardInterrupt\n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", + "ename": "TypeError", + "evalue": "'generator' object does not support item assignment", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[2], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mtranslate\u001b[39;00m \u001b[39mimport\u001b[39;00m yield_untranslated\n\u001b[0;32m----> 3\u001b[0m \u001b[39mfor\u001b[39;00m val_dict \u001b[39min\u001b[39;00m yield_untranslated():\n\u001b[1;32m 4\u001b[0m \u001b[39mprint\u001b[39m(val_dict)\n", - "File \u001b[0;32m~/projects/twitter_scraper/code/translate.py:54\u001b[0m, in \u001b[0;36myield_untranslated\u001b[0;34m()\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[39mwhile\u001b[39;00m val_dict \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 53\u001b[0m \u001b[39myield\u001b[39;00m val_dict\n\u001b[0;32m---> 54\u001b[0m val_dict \u001b[39m=\u001b[39m get_untranslated()\n", - "File \u001b[0;32m~/projects/twitter_scraper/code/translate.py:32\u001b[0m, in \u001b[0;36mget_untranslated\u001b[0;34m()\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[39mwith\u001b[39;00m Data_table() \u001b[39mas\u001b[39;00m dt:\n\u001b[1;32m 26\u001b[0m query \u001b[39m=\u001b[39m (\n\u001b[1;32m 27\u001b[0m \u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39mSELECT * FROM \u001b[39m\u001b[39m{\u001b[39;00mdt\u001b[39m.\u001b[39m_name\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 28\u001b[0m \u001b[39m'\u001b[39m\u001b[39mWHERE content_en IS NULL \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 29\u001b[0m \u001b[39m'\u001b[39m\u001b[39mORDER BY random() \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 30\u001b[0m \u001b[39m'\u001b[39m\u001b[39mLIMIT 1 \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 31\u001b[0m )\n\u001b[0;32m---> 32\u001b[0m df \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39;49mread_sql_query(query, dt\u001b[39m.\u001b[39;49mcon)\n\u001b[1;32m 33\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(df)\u001b[39m==\u001b[39m\u001b[39m0\u001b[39m: \u001b[39mreturn\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 34\u001b[0m val_dict \u001b[39m=\u001b[39m {\n\u001b[1;32m 35\u001b[0m \u001b[39m'\u001b[39m\u001b[39mid\u001b[39m\u001b[39m'\u001b[39m: df[\u001b[39m'\u001b[39m\u001b[39mid\u001b[39m\u001b[39m'\u001b[39m][\u001b[39m0\u001b[39m],\n\u001b[1;32m 36\u001b[0m \u001b[39m'\u001b[39m\u001b[39mcontent\u001b[39m\u001b[39m'\u001b[39m: df[\u001b[39m'\u001b[39m\u001b[39mcontent\u001b[39m\u001b[39m'\u001b[39m][\u001b[39m0\u001b[39m]\n\u001b[1;32m 37\u001b[0m }\n", - "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py:397\u001b[0m, in \u001b[0;36mread_sql_query\u001b[0;34m(sql, con, index_col, coerce_float, params, parse_dates, chunksize, dtype)\u001b[0m\n\u001b[1;32m 339\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 340\u001b[0m \u001b[39mRead SQL query into a DataFrame.\u001b[39;00m\n\u001b[1;32m 341\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 394\u001b[0m \u001b[39mparameter will be converted to UTC.\u001b[39;00m\n\u001b[1;32m 395\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 396\u001b[0m pandas_sql \u001b[39m=\u001b[39m pandasSQL_builder(con)\n\u001b[0;32m--> 397\u001b[0m \u001b[39mreturn\u001b[39;00m pandas_sql\u001b[39m.\u001b[39;49mread_query(\n\u001b[1;32m 398\u001b[0m sql,\n\u001b[1;32m 399\u001b[0m index_col\u001b[39m=\u001b[39;49mindex_col,\n\u001b[1;32m 400\u001b[0m params\u001b[39m=\u001b[39;49mparams,\n\u001b[1;32m 401\u001b[0m coerce_float\u001b[39m=\u001b[39;49mcoerce_float,\n\u001b[1;32m 402\u001b[0m parse_dates\u001b[39m=\u001b[39;49mparse_dates,\n\u001b[1;32m 403\u001b[0m chunksize\u001b[39m=\u001b[39;49mchunksize,\n\u001b[1;32m 404\u001b[0m dtype\u001b[39m=\u001b[39;49mdtype,\n\u001b[1;32m 405\u001b[0m )\n", - "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py:1560\u001b[0m, in \u001b[0;36mSQLDatabase.read_query\u001b[0;34m(self, sql, index_col, coerce_float, parse_dates, params, chunksize, dtype)\u001b[0m\n\u001b[1;32m 1512\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1513\u001b[0m \u001b[39mRead SQL query into a DataFrame.\u001b[39;00m\n\u001b[1;32m 1514\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1556\u001b[0m \n\u001b[1;32m 1557\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1558\u001b[0m args \u001b[39m=\u001b[39m _convert_params(sql, params)\n\u001b[0;32m-> 1560\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mexecute(\u001b[39m*\u001b[39;49margs)\n\u001b[1;32m 1561\u001b[0m columns \u001b[39m=\u001b[39m result\u001b[39m.\u001b[39mkeys()\n\u001b[1;32m 1563\u001b[0m \u001b[39mif\u001b[39;00m chunksize \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n", - "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py:1405\u001b[0m, in \u001b[0;36mSQLDatabase.execute\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1403\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mexecute\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m 1404\u001b[0m \u001b[39m\"\"\"Simple passthrough to SQLAlchemy connectable\"\"\"\u001b[39;00m\n\u001b[0;32m-> 1405\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconnectable\u001b[39m.\u001b[39;49mexecution_options()\u001b[39m.\u001b[39;49mexecute(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n", - "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1365\u001b[0m, in \u001b[0;36mConnection.execute\u001b[0;34m(self, statement, *multiparams, **params)\u001b[0m\n\u001b[1;32m 1356\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(statement, util\u001b[39m.\u001b[39mstring_types):\n\u001b[1;32m 1357\u001b[0m util\u001b[39m.\u001b[39mwarn_deprecated_20(\n\u001b[1;32m 1358\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mPassing a string to Connection.execute() is \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 1359\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mdeprecated and will be removed in version 2.0. Use the \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1362\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mdriver-level SQL string.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 1363\u001b[0m )\n\u001b[0;32m-> 1365\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_exec_driver_sql(\n\u001b[1;32m 1366\u001b[0m statement,\n\u001b[1;32m 1367\u001b[0m multiparams,\n\u001b[1;32m 1368\u001b[0m params,\n\u001b[1;32m 1369\u001b[0m _EMPTY_EXECUTION_OPTS,\n\u001b[1;32m 1370\u001b[0m future\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 1371\u001b[0m )\n\u001b[1;32m 1373\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 1374\u001b[0m meth \u001b[39m=\u001b[39m statement\u001b[39m.\u001b[39m_execute_on_connection\n", - "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1669\u001b[0m, in \u001b[0;36mConnection._exec_driver_sql\u001b[0;34m(self, statement, multiparams, params, execution_options, future)\u001b[0m\n\u001b[1;32m 1659\u001b[0m (\n\u001b[1;32m 1660\u001b[0m statement,\n\u001b[1;32m 1661\u001b[0m distilled_params,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1665\u001b[0m statement, distilled_parameters, execution_options\n\u001b[1;32m 1666\u001b[0m )\n\u001b[1;32m 1668\u001b[0m dialect \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdialect\n\u001b[0;32m-> 1669\u001b[0m ret \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_execute_context(\n\u001b[1;32m 1670\u001b[0m dialect,\n\u001b[1;32m 1671\u001b[0m dialect\u001b[39m.\u001b[39;49mexecution_ctx_cls\u001b[39m.\u001b[39;49m_init_statement,\n\u001b[1;32m 1672\u001b[0m statement,\n\u001b[1;32m 1673\u001b[0m distilled_parameters,\n\u001b[1;32m 1674\u001b[0m execution_options,\n\u001b[1;32m 1675\u001b[0m statement,\n\u001b[1;32m 1676\u001b[0m distilled_parameters,\n\u001b[1;32m 1677\u001b[0m )\n\u001b[1;32m 1679\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m future:\n\u001b[1;32m 1680\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_has_events \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mengine\u001b[39m.\u001b[39m_has_events:\n", - "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1943\u001b[0m, in \u001b[0;36mConnection._execute_context\u001b[0;34m(self, dialect, constructor, statement, parameters, execution_options, *args, **kw)\u001b[0m\n\u001b[1;32m 1940\u001b[0m branched\u001b[39m.\u001b[39mclose()\n\u001b[1;32m 1942\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m-> 1943\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_handle_dbapi_exception(\n\u001b[1;32m 1944\u001b[0m e, statement, parameters, cursor, context\n\u001b[1;32m 1945\u001b[0m )\n\u001b[1;32m 1947\u001b[0m \u001b[39mreturn\u001b[39;00m result\n", - "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:2128\u001b[0m, in \u001b[0;36mConnection._handle_dbapi_exception\u001b[0;34m(self, e, statement, parameters, cursor, context)\u001b[0m\n\u001b[1;32m 2124\u001b[0m util\u001b[39m.\u001b[39mraise_(\n\u001b[1;32m 2125\u001b[0m sqlalchemy_exception, with_traceback\u001b[39m=\u001b[39mexc_info[\u001b[39m2\u001b[39m], from_\u001b[39m=\u001b[39me\n\u001b[1;32m 2126\u001b[0m )\n\u001b[1;32m 2127\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 2128\u001b[0m util\u001b[39m.\u001b[39;49mraise_(exc_info[\u001b[39m1\u001b[39;49m], with_traceback\u001b[39m=\u001b[39;49mexc_info[\u001b[39m2\u001b[39;49m])\n\u001b[1;32m 2130\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 2131\u001b[0m \u001b[39mdel\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_reentrant_error\n", - "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py:211\u001b[0m, in \u001b[0;36mraise_\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m 208\u001b[0m exception\u001b[39m.\u001b[39m__cause__ \u001b[39m=\u001b[39m replace_context\n\u001b[1;32m 210\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 211\u001b[0m \u001b[39mraise\u001b[39;00m exception\n\u001b[1;32m 212\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 213\u001b[0m \u001b[39m# credit to\u001b[39;00m\n\u001b[1;32m 214\u001b[0m \u001b[39m# https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/\u001b[39;00m\n\u001b[1;32m 215\u001b[0m \u001b[39m# as the __traceback__ object creates a cycle\u001b[39;00m\n\u001b[1;32m 216\u001b[0m \u001b[39mdel\u001b[39;00m exception, replace_context, from_, with_traceback\n", - "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1900\u001b[0m, in \u001b[0;36mConnection._execute_context\u001b[0;34m(self, dialect, constructor, statement, parameters, execution_options, *args, **kw)\u001b[0m\n\u001b[1;32m 1898\u001b[0m \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m 1899\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m evt_handled:\n\u001b[0;32m-> 1900\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdialect\u001b[39m.\u001b[39;49mdo_execute(\n\u001b[1;32m 1901\u001b[0m cursor, statement, parameters, context\n\u001b[1;32m 1902\u001b[0m )\n\u001b[1;32m 1904\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_has_events \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mengine\u001b[39m.\u001b[39m_has_events:\n\u001b[1;32m 1905\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdispatch\u001b[39m.\u001b[39mafter_cursor_execute(\n\u001b[1;32m 1906\u001b[0m \u001b[39mself\u001b[39m,\n\u001b[1;32m 1907\u001b[0m cursor,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1911\u001b[0m context\u001b[39m.\u001b[39mexecutemany,\n\u001b[1;32m 1912\u001b[0m )\n", - "File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py:736\u001b[0m, in \u001b[0;36mDefaultDialect.do_execute\u001b[0;34m(self, cursor, statement, parameters, context)\u001b[0m\n\u001b[1;32m 735\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdo_execute\u001b[39m(\u001b[39mself\u001b[39m, cursor, statement, parameters, context\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m):\n\u001b[0;32m--> 736\u001b[0m cursor\u001b[39m.\u001b[39;49mexecute(statement, parameters)\n", - "File \u001b[0;32m/usr/lib/python3.8/encodings/utf_8.py:15\u001b[0m, in \u001b[0;36mdecode\u001b[0;34m(input, errors)\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[39m### Codec APIs\u001b[39;00m\n\u001b[1;32m 13\u001b[0m encode \u001b[39m=\u001b[39m codecs\u001b[39m.\u001b[39mutf_8_encode\n\u001b[0;32m---> 15\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdecode\u001b[39m(\u001b[39minput\u001b[39m, errors\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mstrict\u001b[39m\u001b[39m'\u001b[39m):\n\u001b[1;32m 16\u001b[0m \u001b[39mreturn\u001b[39;00m codecs\u001b[39m.\u001b[39mutf_8_decode(\u001b[39minput\u001b[39m, errors, \u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m 18\u001b[0m \u001b[39mclass\u001b[39;00m \u001b[39mIncrementalEncoder\u001b[39;00m(codecs\u001b[39m.\u001b[39mIncrementalEncoder):\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[2], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mtranslate\u001b[39;00m \u001b[39mimport\u001b[39;00m yield_untranslated, process_translate\n\u001b[1;32m 2\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mmultiprocessing\u001b[39;00m \u001b[39mas\u001b[39;00m \u001b[39mmp\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m process_translate(yield_untranslated())\n", + "File \u001b[0;32m~/projects/twitter_scraper/code/translate.py:46\u001b[0m, in \u001b[0;36mprocess_translate\u001b[0;34m(val_dict)\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 45\u001b[0m content_en \u001b[39m=\u001b[39m \u001b[39m'\u001b[39m\u001b[39m'\u001b[39m\n\u001b[0;32m---> 46\u001b[0m val_dict[\u001b[39m'\u001b[39;49m\u001b[39mcontent_en\u001b[39;49m\u001b[39m'\u001b[39;49m] \u001b[39m=\u001b[39m content_en\n\u001b[1;32m 47\u001b[0m \u001b[39m# upload to database\u001b[39;00m\n\u001b[1;32m 48\u001b[0m insert_translated(val_dict)\n", + "\u001b[0;31mTypeError\u001b[0m: 'generator' object does not support item assignment" ] } ], "source": [ - "from translate import yield_untranslated\n", + "from translate import yield_untranslated, process_translate\n", + "import multiprocessing as mp\n", "\n", - "for val_dict in yield_untranslated():\n", - " print(val_dict)" + "process_translate(yield_untranslated())" ] }, { @@ -150,7 +72,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.8.10 (default, Nov 14 2022, 12:59:47) \n[GCC 9.4.0]" }, "orig_nbformat": 4, "vscode": { diff --git a/code/translate.py b/code/translate.py index 2cdc3c5..832351a 100644 --- a/code/translate.py +++ b/code/translate.py @@ -2,9 +2,13 @@ from langdetect import detect from google_trans_new import google_translator import pandas as pd import multiprocessing as mp +import numpy as np +from dotenv import load_dotenv +from tqdm import tqdm from database import Data_table from database import insert_translated +from clean_data import clean_content def translate(text, target_lang='en'): try: @@ -32,44 +36,26 @@ def get_untranslated(): df = pd.read_sql_query(query, dt.con) if len(df)==0: return None val_dict = { - 'id': df['id'][0], - 'content': df['content'][0] + 'id': int(df['id'][0]), + 'content': str(df['content'][0]) } return val_dict -def process_translate(val_dict): - # translate - try: - content_en = translate(val_dict['content']) - except Exception as e: - content_en = '' - val_dict['content_en'] = content_en - # upload to database - insert_translated(val_dict) - -def yield_untranslated(): - val_dict = get_untranslated() - while val_dict is not None: - yield val_dict +def continously_translate(): + while True: val_dict = get_untranslated() - -# def continued_translation(): -# while True: -# # get untranslated -# val_dict = get_untranslated() -# if val_dict is None: break -# # translate -# try: -# content_en = translate(val_dict['content']) -# except Exception as e: -# content_en = '' -# val_dict['content_en'] = content_en -# # upload to database -# insert_translated(val_dict) - -def translate_all_fast(): - with mp.get_context('spawn').Pool(mp.cpu_count()-1) as p: - list(p.map(process_translate, yield_untranslated())) + if val_dict is None: break + # clean + content_clean = clean_content(val_dict['content']) + # translate + try: + content_en = translate(content_clean) + except Exception as e: + content_en = '' + val_dict['content_en'] = content_en + # upload to database + insert_translated(val_dict) + print('no more untranslated content located') if __name__ == '__main__': - translate_all_fast() + continously_translate()