updated code to run on multiple hosts at the same time
This commit is contained in:
brian
+10
-88
@@ -29,101 +29,23 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
"name": "stdout",
|
"ename": "TypeError",
|
||||||
"output_type": "stream",
|
"evalue": "'generator' object does not support item assignment",
|
||||||
"text": [
|
|
||||||
"{'id': 151028, 'content': 'Åbner i vinterferien: Nyt børnemuseum holder åbningsfest med Sigurd Barrett Odense: Har man nysgerrige børn med sa… https://t.co/wsy3JBodsw'}\n",
|
|
||||||
"{'id': 89329, 'content': 'Guitarkabinet, Marshall Jcm 900, 300 W, Lækkert..., 1.500 kr.,- i Odense V 5200 - https://t.co/UY0x8rTnKH https://t.co/aCOd9CmE80'}\n",
|
|
||||||
"{'id': 123474, 'content': 'ODEON søger kreativ bager https://t.co/NznIkPH6qc Job i Odense'}\n",
|
|
||||||
"{'id': 121074, 'content': 'Kommunikationsansvarlig for UngEnergi Odense (UngEnergi) https://t.co/wMnSJ9bifG Job i Odense'}\n",
|
|
||||||
"{'id': 49585, 'content': '37 Year Ago\\n#LiverpoolFC #LFC #TheReds #Liverpool #BKOdense #BKO #TheStriped #Odense https://t.co/0wJxft6RCD'}\n",
|
|
||||||
"{'id': 112745, 'content': 'Regnskabschef med IT-ansvar https://t.co/2orZw0VdQA Job i Odense'}\n",
|
|
||||||
"{'id': 110761, 'content': 'Bugün 2-2 / 3-1 bitebilecek maçlar:\\n\\n1. Leipzig-Köln\\n2-2 (14.60)\\n3-1 (10.60)\\n\\n2. Man City 3-1 Arsenal\\n3. Lyon 3-1St.Etienne (10.20)\\n\\n4. Kopenhag-Odense\\n2-2 (12.50)\\n3-1 (9.25) ⭐️\\n\\n5. Palace-Tottenham\\n2-2 (16.00)\\n1-3 (10.20) ⭐️'}\n",
|
|
||||||
"{'id': 130457, 'content': 'LÆKKER TAPAS MED LÆKKER DAME #myntejensen ❤️ @ Odense, Denmark https://t.co/TwCNXwmDLy'}\n",
|
|
||||||
"{'id': 53302, 'content': 'Elguitar, Jackson SL3X soloist, Sælger min..., 2.000 kr.,- i Odense C 5000 - https://t.co/P9FPoAMA5L https://t.co/WJ3Jolnj8j'}\n",
|
|
||||||
"{'id': 103309, 'content': 'Spillemyndigheden søger #akademisk medarbejder i #Odense. Læs mere på https://t.co/4Ac7RVXazo #jobdk #jobiOdense'}\n",
|
|
||||||
"{'id': 135454, 'content': 'lvl 4 Blastoise raid until 21:31:51 (58m 32s) near Østerbæksvej 83, Odense M. https://t.co/4qRn0maKUu'}\n",
|
|
||||||
"{'id': 146330, 'content': 'Guitarcombo, Two rock Coral, 22 W, Med celestion..., kr. 15.500,- i Odense C 5000 - https://t.co/ATCr7nVXXh https://t.co/Qiia2bDxlh'}\n",
|
|
||||||
"{'id': 115928, 'content': 'Demens i Odense Demens eller Alzheimers sygdom er en folkesygdom og den femte mest hyppige dødsårsag i Danmark.\\nI takt med at antallet og andelen af ældre borgere stiger i Danmark og i Odense Kommune, stiger også ant… https://t.co/S0wFjnWN5c'}\n",
|
|
||||||
"{'id': 138345, 'content': 'Danmarks størreste #sex #dating portal:\\n\\nhttps://t.co/AnhrlR5efT\\n\\n#Danmark #Denmark\\n#Roskilde #Odense #Copenhagen https://t.co/z39Jz41XGW'}\n",
|
|
||||||
"{'id': 88562, 'content': 'Job opening: Director, Centre on Population Dynamics (CPOP), University of Southern Denmark, Odense https://t.co/hMW0WMMgMa Deadline for applications: 20 December 2018'}\n",
|
|
||||||
"{'id': 118162, 'content': 'Dabbing 🙅🏼\\u200d♂️ em Odense, Denmark https://t.co/J5RDVhLD7Z'}\n",
|
|
||||||
"{'id': 60766, 'content': 'Odense Kommune søger #teamleder, sundhed i #Odense. Læs mere på https://t.co/WU9UTz3LnF #jobdk #jobiOdense'}\n",
|
|
||||||
"{'id': 149516, 'content': 'Danmarks størreste #sex #dating portal:\\n\\nhttps://t.co/AnhrlR5efT\\n\\n#Danmark #Denmark\\n#Roskilde #Odense #Copenhagen https://t.co/6IR7VqoeUp'}\n",
|
|
||||||
"{'id': 134039, 'content': '28-årig fra Odense dømt for drugrape af to kvinder https://t.co/XrJgnVTzt4'}\n",
|
|
||||||
"{'id': 63395, 'content': 'Inspiring and interesting talk\\n #ecgbl2019 @odense about #seriousgames for girls @serenagame #diversity #gbl #gamedev @p_spangen https://t.co/EX7wYoQmBy'}\n",
|
|
||||||
"{'id': 123129, 'content': 'Klar i nyhedsstudiet med dagens fynske nyheder. God efterårsferie derude 😀 #tv2fyn #odense #tv #nyheder https://t.co/0cUPo15Q4n'}\n",
|
|
||||||
"{'id': 78825, 'content': 'Today all the houses in my street had smoke detectors installed. #bettersafethansorry #smokedetector #fire #safetyfirst #Odense #Denmark @ Odense, Denmark https://t.co/s8D5Ikd4D5'}\n",
|
|
||||||
"{'id': 18288, 'content': 'Highlights: SønderjyskE – Odense Bulldogs https://t.co/W0fPazU1Pr #Highlights #hockeymagasinetdk'}\n",
|
|
||||||
"{'id': 115120, 'content': 'Seniorskolen i Nørre Aaby klar til ny sæson Nr. Aaby: Seniorskolen, som holder til i Nr. Aaby Sognehus og mødes udvalgte onsdag klokken 14, er klar til en ny sæson.\\n10. januar handler det om strejftog i Højskolesangbogen med leder af Odense Mot… … https://t.co/VSU5Je8Y4x'}\n",
|
|
||||||
"{'id': 125121, 'content': 'Odense Kommune søger #områdechef i #Odense. Læs mere på https://t.co/pe9H1SsO3Q #jobdk #jobiOdense'}\n",
|
|
||||||
"{'id': 84382, 'content': 'Someone from Odense, Denmark 🇩🇰 ordered this Military Watch Quartz Leather.\\n\\n 🛒 You too, you can get yours at ➡ https://t.co/JIt9HqnezT . https://t.co/8oHRnkw5Lm'}\n",
|
|
||||||
"{'id': 71585, 'content': '#zulucs Er først lige kommet på. Har I et par kommentarer til det såkaldte DM i esport i Odense i dag? Er det fedt for CS med DM? Hvad med forholdet mellem lange onlineturneringer og korte LANs? Er der overhovedet noget fest i et CS-DM? Og tillykke til Aab #hfesport'}\n",
|
|
||||||
"{'id': 125496, 'content': 'Soccer #Livescore: (DEN-3R) #FCAmager vs #Odense: 1-1. 2nd Half Started ...'}\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "stderr",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Traceback (most recent call last):\n",
|
|
||||||
" File \"/home/brian/projects/twitter_scraper/code/translate.py\", line 32, in get_untranslated\n",
|
|
||||||
" df = pd.read_sql_query(query, dt.con)\n",
|
|
||||||
" File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py\", line 397, in read_sql_query\n",
|
|
||||||
" return pandas_sql.read_query(\n",
|
|
||||||
" File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py\", line 1560, in read_query\n",
|
|
||||||
" result = self.execute(*args)\n",
|
|
||||||
" File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py\", line 1405, in execute\n",
|
|
||||||
" return self.connectable.execution_options().execute(*args, **kwargs)\n",
|
|
||||||
" File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 1365, in execute\n",
|
|
||||||
" return self._exec_driver_sql(\n",
|
|
||||||
" File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 1669, in _exec_driver_sql\n",
|
|
||||||
" ret = self._execute_context(\n",
|
|
||||||
" File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 1943, in _execute_context\n",
|
|
||||||
" self._handle_dbapi_exception(\n",
|
|
||||||
" File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 2128, in _handle_dbapi_exception\n",
|
|
||||||
" util.raise_(exc_info[1], with_traceback=exc_info[2])\n",
|
|
||||||
" File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py\", line 211, in raise_\n",
|
|
||||||
" raise exception\n",
|
|
||||||
" File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py\", line 1900, in _execute_context\n",
|
|
||||||
" self.dialect.do_execute(\n",
|
|
||||||
" File \"/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py\", line 736, in do_execute\n",
|
|
||||||
" cursor.execute(statement, parameters)\n",
|
|
||||||
" File \"/usr/lib/python3.8/encodings/utf_8.py\", line 15, in decode\n",
|
|
||||||
" def decode(input, errors='strict'):\n",
|
|
||||||
"KeyboardInterrupt\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"ename": "KeyboardInterrupt",
|
|
||||||
"evalue": "",
|
|
||||||
"output_type": "error",
|
"output_type": "error",
|
||||||
"traceback": [
|
"traceback": [
|
||||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||||
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
|
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
|
||||||
"Cell \u001b[0;32mIn[2], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mtranslate\u001b[39;00m \u001b[39mimport\u001b[39;00m yield_untranslated\n\u001b[0;32m----> 3\u001b[0m \u001b[39mfor\u001b[39;00m val_dict \u001b[39min\u001b[39;00m yield_untranslated():\n\u001b[1;32m 4\u001b[0m \u001b[39mprint\u001b[39m(val_dict)\n",
|
"Cell \u001b[0;32mIn[2], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mtranslate\u001b[39;00m \u001b[39mimport\u001b[39;00m yield_untranslated, process_translate\n\u001b[1;32m 2\u001b[0m \u001b[39mimport\u001b[39;00m \u001b[39mmultiprocessing\u001b[39;00m \u001b[39mas\u001b[39;00m \u001b[39mmp\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m process_translate(yield_untranslated())\n",
|
||||||
"File \u001b[0;32m~/projects/twitter_scraper/code/translate.py:54\u001b[0m, in \u001b[0;36myield_untranslated\u001b[0;34m()\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[39mwhile\u001b[39;00m val_dict \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 53\u001b[0m \u001b[39myield\u001b[39;00m val_dict\n\u001b[0;32m---> 54\u001b[0m val_dict \u001b[39m=\u001b[39m get_untranslated()\n",
|
"File \u001b[0;32m~/projects/twitter_scraper/code/translate.py:46\u001b[0m, in \u001b[0;36mprocess_translate\u001b[0;34m(val_dict)\u001b[0m\n\u001b[1;32m 44\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 45\u001b[0m content_en \u001b[39m=\u001b[39m \u001b[39m'\u001b[39m\u001b[39m'\u001b[39m\n\u001b[0;32m---> 46\u001b[0m val_dict[\u001b[39m'\u001b[39;49m\u001b[39mcontent_en\u001b[39;49m\u001b[39m'\u001b[39;49m] \u001b[39m=\u001b[39m content_en\n\u001b[1;32m 47\u001b[0m \u001b[39m# upload to database\u001b[39;00m\n\u001b[1;32m 48\u001b[0m insert_translated(val_dict)\n",
|
||||||
"File \u001b[0;32m~/projects/twitter_scraper/code/translate.py:32\u001b[0m, in \u001b[0;36mget_untranslated\u001b[0;34m()\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[39mwith\u001b[39;00m Data_table() \u001b[39mas\u001b[39;00m dt:\n\u001b[1;32m 26\u001b[0m query \u001b[39m=\u001b[39m (\n\u001b[1;32m 27\u001b[0m \u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39mSELECT * FROM \u001b[39m\u001b[39m{\u001b[39;00mdt\u001b[39m.\u001b[39m_name\u001b[39m}\u001b[39;00m\u001b[39m \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 28\u001b[0m \u001b[39m'\u001b[39m\u001b[39mWHERE content_en IS NULL \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 29\u001b[0m \u001b[39m'\u001b[39m\u001b[39mORDER BY random() \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 30\u001b[0m \u001b[39m'\u001b[39m\u001b[39mLIMIT 1 \u001b[39m\u001b[39m'\u001b[39m\n\u001b[1;32m 31\u001b[0m )\n\u001b[0;32m---> 32\u001b[0m df \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39;49mread_sql_query(query, dt\u001b[39m.\u001b[39;49mcon)\n\u001b[1;32m 33\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(df)\u001b[39m==\u001b[39m\u001b[39m0\u001b[39m: \u001b[39mreturn\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 34\u001b[0m val_dict \u001b[39m=\u001b[39m {\n\u001b[1;32m 35\u001b[0m \u001b[39m'\u001b[39m\u001b[39mid\u001b[39m\u001b[39m'\u001b[39m: df[\u001b[39m'\u001b[39m\u001b[39mid\u001b[39m\u001b[39m'\u001b[39m][\u001b[39m0\u001b[39m],\n\u001b[1;32m 36\u001b[0m \u001b[39m'\u001b[39m\u001b[39mcontent\u001b[39m\u001b[39m'\u001b[39m: df[\u001b[39m'\u001b[39m\u001b[39mcontent\u001b[39m\u001b[39m'\u001b[39m][\u001b[39m0\u001b[39m]\n\u001b[1;32m 37\u001b[0m }\n",
|
"\u001b[0;31mTypeError\u001b[0m: 'generator' object does not support item assignment"
|
||||||
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py:397\u001b[0m, in \u001b[0;36mread_sql_query\u001b[0;34m(sql, con, index_col, coerce_float, params, parse_dates, chunksize, dtype)\u001b[0m\n\u001b[1;32m 339\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 340\u001b[0m \u001b[39mRead SQL query into a DataFrame.\u001b[39;00m\n\u001b[1;32m 341\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 394\u001b[0m \u001b[39mparameter will be converted to UTC.\u001b[39;00m\n\u001b[1;32m 395\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 396\u001b[0m pandas_sql \u001b[39m=\u001b[39m pandasSQL_builder(con)\n\u001b[0;32m--> 397\u001b[0m \u001b[39mreturn\u001b[39;00m pandas_sql\u001b[39m.\u001b[39;49mread_query(\n\u001b[1;32m 398\u001b[0m sql,\n\u001b[1;32m 399\u001b[0m index_col\u001b[39m=\u001b[39;49mindex_col,\n\u001b[1;32m 400\u001b[0m params\u001b[39m=\u001b[39;49mparams,\n\u001b[1;32m 401\u001b[0m coerce_float\u001b[39m=\u001b[39;49mcoerce_float,\n\u001b[1;32m 402\u001b[0m parse_dates\u001b[39m=\u001b[39;49mparse_dates,\n\u001b[1;32m 403\u001b[0m chunksize\u001b[39m=\u001b[39;49mchunksize,\n\u001b[1;32m 404\u001b[0m dtype\u001b[39m=\u001b[39;49mdtype,\n\u001b[1;32m 405\u001b[0m )\n",
|
|
||||||
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py:1560\u001b[0m, in \u001b[0;36mSQLDatabase.read_query\u001b[0;34m(self, sql, index_col, coerce_float, parse_dates, params, chunksize, dtype)\u001b[0m\n\u001b[1;32m 1512\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1513\u001b[0m \u001b[39mRead SQL query into a DataFrame.\u001b[39;00m\n\u001b[1;32m 1514\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1556\u001b[0m \n\u001b[1;32m 1557\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1558\u001b[0m args \u001b[39m=\u001b[39m _convert_params(sql, params)\n\u001b[0;32m-> 1560\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mexecute(\u001b[39m*\u001b[39;49margs)\n\u001b[1;32m 1561\u001b[0m columns \u001b[39m=\u001b[39m result\u001b[39m.\u001b[39mkeys()\n\u001b[1;32m 1563\u001b[0m \u001b[39mif\u001b[39;00m chunksize \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
|
|
||||||
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py:1405\u001b[0m, in \u001b[0;36mSQLDatabase.execute\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1403\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mexecute\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m 1404\u001b[0m \u001b[39m\"\"\"Simple passthrough to SQLAlchemy connectable\"\"\"\u001b[39;00m\n\u001b[0;32m-> 1405\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconnectable\u001b[39m.\u001b[39;49mexecution_options()\u001b[39m.\u001b[39;49mexecute(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
|
|
||||||
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1365\u001b[0m, in \u001b[0;36mConnection.execute\u001b[0;34m(self, statement, *multiparams, **params)\u001b[0m\n\u001b[1;32m 1356\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(statement, util\u001b[39m.\u001b[39mstring_types):\n\u001b[1;32m 1357\u001b[0m util\u001b[39m.\u001b[39mwarn_deprecated_20(\n\u001b[1;32m 1358\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mPassing a string to Connection.execute() is \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 1359\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mdeprecated and will be removed in version 2.0. Use the \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1362\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mdriver-level SQL string.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 1363\u001b[0m )\n\u001b[0;32m-> 1365\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_exec_driver_sql(\n\u001b[1;32m 1366\u001b[0m statement,\n\u001b[1;32m 1367\u001b[0m multiparams,\n\u001b[1;32m 1368\u001b[0m params,\n\u001b[1;32m 1369\u001b[0m _EMPTY_EXECUTION_OPTS,\n\u001b[1;32m 1370\u001b[0m future\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 1371\u001b[0m )\n\u001b[1;32m 1373\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 1374\u001b[0m meth \u001b[39m=\u001b[39m statement\u001b[39m.\u001b[39m_execute_on_connection\n",
|
|
||||||
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1669\u001b[0m, in \u001b[0;36mConnection._exec_driver_sql\u001b[0;34m(self, statement, multiparams, params, execution_options, future)\u001b[0m\n\u001b[1;32m 1659\u001b[0m (\n\u001b[1;32m 1660\u001b[0m statement,\n\u001b[1;32m 1661\u001b[0m distilled_params,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1665\u001b[0m statement, distilled_parameters, execution_options\n\u001b[1;32m 1666\u001b[0m )\n\u001b[1;32m 1668\u001b[0m dialect \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdialect\n\u001b[0;32m-> 1669\u001b[0m ret \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_execute_context(\n\u001b[1;32m 1670\u001b[0m dialect,\n\u001b[1;32m 1671\u001b[0m dialect\u001b[39m.\u001b[39;49mexecution_ctx_cls\u001b[39m.\u001b[39;49m_init_statement,\n\u001b[1;32m 1672\u001b[0m statement,\n\u001b[1;32m 1673\u001b[0m distilled_parameters,\n\u001b[1;32m 1674\u001b[0m execution_options,\n\u001b[1;32m 1675\u001b[0m statement,\n\u001b[1;32m 1676\u001b[0m distilled_parameters,\n\u001b[1;32m 1677\u001b[0m )\n\u001b[1;32m 1679\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m future:\n\u001b[1;32m 1680\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_has_events \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mengine\u001b[39m.\u001b[39m_has_events:\n",
|
|
||||||
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1943\u001b[0m, in \u001b[0;36mConnection._execute_context\u001b[0;34m(self, dialect, constructor, statement, parameters, execution_options, *args, **kw)\u001b[0m\n\u001b[1;32m 1940\u001b[0m branched\u001b[39m.\u001b[39mclose()\n\u001b[1;32m 1942\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m-> 1943\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_handle_dbapi_exception(\n\u001b[1;32m 1944\u001b[0m e, statement, parameters, cursor, context\n\u001b[1;32m 1945\u001b[0m )\n\u001b[1;32m 1947\u001b[0m \u001b[39mreturn\u001b[39;00m result\n",
|
|
||||||
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:2128\u001b[0m, in \u001b[0;36mConnection._handle_dbapi_exception\u001b[0;34m(self, e, statement, parameters, cursor, context)\u001b[0m\n\u001b[1;32m 2124\u001b[0m util\u001b[39m.\u001b[39mraise_(\n\u001b[1;32m 2125\u001b[0m sqlalchemy_exception, with_traceback\u001b[39m=\u001b[39mexc_info[\u001b[39m2\u001b[39m], from_\u001b[39m=\u001b[39me\n\u001b[1;32m 2126\u001b[0m )\n\u001b[1;32m 2127\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 2128\u001b[0m util\u001b[39m.\u001b[39;49mraise_(exc_info[\u001b[39m1\u001b[39;49m], with_traceback\u001b[39m=\u001b[39;49mexc_info[\u001b[39m2\u001b[39;49m])\n\u001b[1;32m 2130\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 2131\u001b[0m \u001b[39mdel\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_reentrant_error\n",
|
|
||||||
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py:211\u001b[0m, in \u001b[0;36mraise_\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m 208\u001b[0m exception\u001b[39m.\u001b[39m__cause__ \u001b[39m=\u001b[39m replace_context\n\u001b[1;32m 210\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 211\u001b[0m \u001b[39mraise\u001b[39;00m exception\n\u001b[1;32m 212\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 213\u001b[0m \u001b[39m# credit to\u001b[39;00m\n\u001b[1;32m 214\u001b[0m \u001b[39m# https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/\u001b[39;00m\n\u001b[1;32m 215\u001b[0m \u001b[39m# as the __traceback__ object creates a cycle\u001b[39;00m\n\u001b[1;32m 216\u001b[0m \u001b[39mdel\u001b[39;00m exception, replace_context, from_, with_traceback\n",
|
|
||||||
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1900\u001b[0m, in \u001b[0;36mConnection._execute_context\u001b[0;34m(self, dialect, constructor, statement, parameters, execution_options, *args, **kw)\u001b[0m\n\u001b[1;32m 1898\u001b[0m \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m 1899\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m evt_handled:\n\u001b[0;32m-> 1900\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdialect\u001b[39m.\u001b[39;49mdo_execute(\n\u001b[1;32m 1901\u001b[0m cursor, statement, parameters, context\n\u001b[1;32m 1902\u001b[0m )\n\u001b[1;32m 1904\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_has_events \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mengine\u001b[39m.\u001b[39m_has_events:\n\u001b[1;32m 1905\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdispatch\u001b[39m.\u001b[39mafter_cursor_execute(\n\u001b[1;32m 1906\u001b[0m \u001b[39mself\u001b[39m,\n\u001b[1;32m 1907\u001b[0m cursor,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1911\u001b[0m context\u001b[39m.\u001b[39mexecutemany,\n\u001b[1;32m 1912\u001b[0m )\n",
|
|
||||||
"File \u001b[0;32m~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py:736\u001b[0m, in \u001b[0;36mDefaultDialect.do_execute\u001b[0;34m(self, cursor, statement, parameters, context)\u001b[0m\n\u001b[1;32m 735\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdo_execute\u001b[39m(\u001b[39mself\u001b[39m, cursor, statement, parameters, context\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m):\n\u001b[0;32m--> 736\u001b[0m cursor\u001b[39m.\u001b[39;49mexecute(statement, parameters)\n",
|
|
||||||
"File \u001b[0;32m/usr/lib/python3.8/encodings/utf_8.py:15\u001b[0m, in \u001b[0;36mdecode\u001b[0;34m(input, errors)\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[39m### Codec APIs\u001b[39;00m\n\u001b[1;32m 13\u001b[0m encode \u001b[39m=\u001b[39m codecs\u001b[39m.\u001b[39mutf_8_encode\n\u001b[0;32m---> 15\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdecode\u001b[39m(\u001b[39minput\u001b[39m, errors\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mstrict\u001b[39m\u001b[39m'\u001b[39m):\n\u001b[1;32m 16\u001b[0m \u001b[39mreturn\u001b[39;00m codecs\u001b[39m.\u001b[39mutf_8_decode(\u001b[39minput\u001b[39m, errors, \u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m 18\u001b[0m \u001b[39mclass\u001b[39;00m \u001b[39mIncrementalEncoder\u001b[39;00m(codecs\u001b[39m.\u001b[39mIncrementalEncoder):\n",
|
|
||||||
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"source": [
|
"source": [
|
||||||
"from translate import yield_untranslated\n",
|
"from translate import yield_untranslated, process_translate\n",
|
||||||
|
"import multiprocessing as mp\n",
|
||||||
"\n",
|
"\n",
|
||||||
"for val_dict in yield_untranslated():\n",
|
"process_translate(yield_untranslated())"
|
||||||
" print(val_dict)"
|
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -150,7 +72,7 @@
|
|||||||
"name": "python",
|
"name": "python",
|
||||||
"nbconvert_exporter": "python",
|
"nbconvert_exporter": "python",
|
||||||
"pygments_lexer": "ipython3",
|
"pygments_lexer": "ipython3",
|
||||||
"version": "3.8.10"
|
"version": "3.8.10 (default, Nov 14 2022, 12:59:47) \n[GCC 9.4.0]"
|
||||||
},
|
},
|
||||||
"orig_nbformat": 4,
|
"orig_nbformat": 4,
|
||||||
"vscode": {
|
"vscode": {
|
||||||
|
|||||||
+21
-35
@@ -2,9 +2,13 @@ from langdetect import detect
|
|||||||
from google_trans_new import google_translator
|
from google_trans_new import google_translator
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import multiprocessing as mp
|
import multiprocessing as mp
|
||||||
|
import numpy as np
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
from tqdm import tqdm
|
||||||
|
|
||||||
from database import Data_table
|
from database import Data_table
|
||||||
from database import insert_translated
|
from database import insert_translated
|
||||||
|
from clean_data import clean_content
|
||||||
|
|
||||||
def translate(text, target_lang='en'):
|
def translate(text, target_lang='en'):
|
||||||
try:
|
try:
|
||||||
@@ -32,44 +36,26 @@ def get_untranslated():
|
|||||||
df = pd.read_sql_query(query, dt.con)
|
df = pd.read_sql_query(query, dt.con)
|
||||||
if len(df)==0: return None
|
if len(df)==0: return None
|
||||||
val_dict = {
|
val_dict = {
|
||||||
'id': df['id'][0],
|
'id': int(df['id'][0]),
|
||||||
'content': df['content'][0]
|
'content': str(df['content'][0])
|
||||||
}
|
}
|
||||||
return val_dict
|
return val_dict
|
||||||
|
|
||||||
def process_translate(val_dict):
|
def continously_translate():
|
||||||
# translate
|
while True:
|
||||||
try:
|
|
||||||
content_en = translate(val_dict['content'])
|
|
||||||
except Exception as e:
|
|
||||||
content_en = ''
|
|
||||||
val_dict['content_en'] = content_en
|
|
||||||
# upload to database
|
|
||||||
insert_translated(val_dict)
|
|
||||||
|
|
||||||
def yield_untranslated():
|
|
||||||
val_dict = get_untranslated()
|
|
||||||
while val_dict is not None:
|
|
||||||
yield val_dict
|
|
||||||
val_dict = get_untranslated()
|
val_dict = get_untranslated()
|
||||||
|
if val_dict is None: break
|
||||||
# def continued_translation():
|
# clean
|
||||||
# while True:
|
content_clean = clean_content(val_dict['content'])
|
||||||
# # get untranslated
|
# translate
|
||||||
# val_dict = get_untranslated()
|
try:
|
||||||
# if val_dict is None: break
|
content_en = translate(content_clean)
|
||||||
# # translate
|
except Exception as e:
|
||||||
# try:
|
content_en = ''
|
||||||
# content_en = translate(val_dict['content'])
|
val_dict['content_en'] = content_en
|
||||||
# except Exception as e:
|
# upload to database
|
||||||
# content_en = ''
|
insert_translated(val_dict)
|
||||||
# val_dict['content_en'] = content_en
|
print('no more untranslated content located')
|
||||||
# # upload to database
|
|
||||||
# insert_translated(val_dict)
|
|
||||||
|
|
||||||
def translate_all_fast():
|
|
||||||
with mp.get_context('spawn').Pool(mp.cpu_count()-1) as p:
|
|
||||||
list(p.map(process_translate, yield_untranslated()))
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
translate_all_fast()
|
continously_translate()
|
||||||
|
|||||||
Reference in New Issue
Block a user