twitter_scraper/code/translate.ipynb

In [17]:

from translate import get_untranslated

get_untranslated()

Out [17]:

{'id': 108482,
 'content': 'Chat, flirt, love!\ntilmeld dig GRATIS nu:\n\nhttps://t.co/5Vk2bxjdqm\n\n#Danmark #Århus #Odense #Copenhagen #Denmark https://t.co/3XbIv2jImj'}

In [2]:

from translate import yield_untranslated

for val_dict in yield_untranslated():
    print(val_dict)

{'id': 151028, 'content': 'Åbner i vinterferien: Nyt børnemuseum holder åbningsfest med Sigurd Barrett Odense: Har man nysgerrige børn med sa… https://t.co/wsy3JBodsw'}
{'id': 89329, 'content': 'Guitarkabinet, Marshall  Jcm 900, 300 W, Lækkert..., 1.500 kr.,- i Odense V 5200 - https://t.co/UY0x8rTnKH https://t.co/aCOd9CmE80'}
{'id': 123474, 'content': 'ODEON søger kreativ bager https://t.co/NznIkPH6qc Job i Odense'}
{'id': 121074, 'content': 'Kommunikationsansvarlig for UngEnergi Odense (UngEnergi) https://t.co/wMnSJ9bifG Job i Odense'}
{'id': 49585, 'content': '37 Year Ago\n#LiverpoolFC #LFC #TheReds #Liverpool #BKOdense #BKO #TheStriped #Odense https://t.co/0wJxft6RCD'}
{'id': 112745, 'content': 'Regnskabschef med IT-ansvar https://t.co/2orZw0VdQA Job i Odense'}
{'id': 110761, 'content': 'Bugün 2-2 / 3-1 bitebilecek maçlar:\n\n1. Leipzig-Köln\n2-2 (14.60)\n3-1 (10.60)\n\n2. Man City 3-1 Arsenal\n3. Lyon 3-1St.Etienne (10.20)\n\n4. Kopenhag-Odense\n2-2 (12.50)\n3-1 (9.25) ⭐️\n\n5. Palace-Tottenham\n2-2 (16.00)\n1-3 (10.20) ⭐️'}
{'id': 130457, 'content': 'LÆKKER TAPAS MED LÆKKER DAME #myntejensen ❤️ @ Odense, Denmark https://t.co/TwCNXwmDLy'}
{'id': 53302, 'content': 'Elguitar, Jackson SL3X soloist, Sælger min..., 2.000 kr.,- i Odense C 5000 - https://t.co/P9FPoAMA5L https://t.co/WJ3Jolnj8j'}
{'id': 103309, 'content': 'Spillemyndigheden søger #akademisk medarbejder i #Odense. Læs mere på https://t.co/4Ac7RVXazo #jobdk #jobiOdense'}
{'id': 135454, 'content': 'lvl 4 Blastoise raid until 21:31:51 (58m 32s) near Østerbæksvej 83, Odense M. https://t.co/4qRn0maKUu'}
{'id': 146330, 'content': 'Guitarcombo, Two rock Coral, 22 W, Med celestion..., kr. 15.500,- i Odense C 5000 - https://t.co/ATCr7nVXXh https://t.co/Qiia2bDxlh'}
{'id': 115928, 'content': 'Demens i Odense Demens eller Alzheimers sygdom er en folkesygdom og den femte mest hyppige dødsårsag i Danmark.\nI takt med at antallet og andelen af ældre borgere stiger i Danmark og i Odense Kommune, stiger også ant… https://t.co/S0wFjnWN5c'}
{'id': 138345, 'content': 'Danmarks størreste #sex #dating portal:\n\nhttps://t.co/AnhrlR5efT\n\n#Danmark #Denmark\n#Roskilde #Odense #Copenhagen https://t.co/z39Jz41XGW'}
{'id': 88562, 'content': 'Job opening:  Director, Centre on Population Dynamics (CPOP), University of Southern Denmark, Odense https://t.co/hMW0WMMgMa Deadline for applications: 20 December 2018'}
{'id': 118162, 'content': 'Dabbing 🙅🏼\u200d♂️ em Odense, Denmark https://t.co/J5RDVhLD7Z'}
{'id': 60766, 'content': 'Odense Kommune søger #teamleder, sundhed i #Odense. Læs mere på https://t.co/WU9UTz3LnF #jobdk #jobiOdense'}
{'id': 149516, 'content': 'Danmarks størreste #sex #dating portal:\n\nhttps://t.co/AnhrlR5efT\n\n#Danmark #Denmark\n#Roskilde #Odense #Copenhagen https://t.co/6IR7VqoeUp'}
{'id': 134039, 'content': '28-årig fra Odense dømt for drugrape af to kvinder https://t.co/XrJgnVTzt4'}
{'id': 63395, 'content': 'Inspiring and interesting talk\n #ecgbl2019 @odense about #seriousgames for girls @serenagame #diversity #gbl #gamedev @p_spangen https://t.co/EX7wYoQmBy'}
{'id': 123129, 'content': 'Klar i nyhedsstudiet med dagens fynske nyheder.       God efterårsferie derude 😀       #tv2fyn #odense #tv #nyheder https://t.co/0cUPo15Q4n'}
{'id': 78825, 'content': 'Today all the houses in my street had smoke detectors installed. #bettersafethansorry #smokedetector #fire #safetyfirst #Odense #Denmark @ Odense, Denmark https://t.co/s8D5Ikd4D5'}
{'id': 18288, 'content': 'Highlights: SønderjyskE – Odense Bulldogs https://t.co/W0fPazU1Pr #Highlights #hockeymagasinetdk'}
{'id': 115120, 'content': 'Seniorskolen i Nørre Aaby klar til ny sæson Nr. Aaby: Seniorskolen, som holder til i Nr. Aaby Sognehus og mødes udvalgte onsdag klokken 14, er klar til en ny sæson.\n10. januar handler det om strejftog i Højskolesangbogen med leder af Odense Mot… … https://t.co/VSU5Je8Y4x'}
{'id': 125121, 'content': 'Odense Kommune søger #områdechef i #Odense. Læs mere på https://t.co/pe9H1SsO3Q #jobdk #jobiOdense'}
{'id': 84382, 'content': 'Someone from Odense, Denmark  🇩🇰 ordered this Military Watch Quartz Leather.\n\n 🛒 You too, you can get yours at ➡ https://t.co/JIt9HqnezT . https://t.co/8oHRnkw5Lm'}
{'id': 71585, 'content': '#zulucs Er først lige kommet på. Har I et par kommentarer til det såkaldte DM i esport i Odense i dag? Er det fedt for CS med DM? Hvad med forholdet mellem lange onlineturneringer og korte LANs? Er der overhovedet noget fest i et CS-DM? Og tillykke til Aab #hfesport'}
{'id': 125496, 'content': 'Soccer #Livescore: (DEN-3R) #FCAmager vs #Odense: 1-1. 2nd Half Started ...'}

Traceback (most recent call last):
  File "/home/brian/projects/twitter_scraper/code/translate.py", line 32, in get_untranslated
    df = pd.read_sql_query(query, dt.con)
  File "/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py", line 397, in read_sql_query
    return pandas_sql.read_query(
  File "/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py", line 1560, in read_query
    result = self.execute(*args)
  File "/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py", line 1405, in execute
    return self.connectable.execution_options().execute(*args, **kwargs)
  File "/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1365, in execute
    return self._exec_driver_sql(
  File "/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1669, in _exec_driver_sql
    ret = self._execute_context(
  File "/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1943, in _execute_context
    self._handle_dbapi_exception(
  File "/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 2128, in _handle_dbapi_exception
    util.raise_(exc_info[1], with_traceback=exc_info[2])
  File "/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 211, in raise_
    raise exception
  File "/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1900, in _execute_context
    self.dialect.do_execute(
  File "/home/brian/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py", line 736, in do_execute
    cursor.execute(statement, parameters)
  File "/usr/lib/python3.8/encodings/utf_8.py", line 15, in decode
    def decode(input, errors='strict'):
KeyboardInterrupt

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
Cell In[2], line 3
      1 from translate import yield_untranslated
----> 3 for val_dict in yield_untranslated():
      4     print(val_dict)

File ~/projects/twitter_scraper/code/translate.py:54, in yield_untranslated()
     52 while val_dict is not None:
     53     yield val_dict
---> 54     val_dict = get_untranslated()

File ~/projects/twitter_scraper/code/translate.py:32, in get_untranslated()
     25 with Data_table() as dt:
     26     query = (
     27         f'SELECT * FROM {dt._name} '
     28         'WHERE content_en IS NULL '
     29         'ORDER BY random() '
     30         'LIMIT 1 '
     31     )
---> 32     df = pd.read_sql_query(query, dt.con)
     33 if len(df)==0: return None
     34 val_dict = {
     35     'id': df['id'][0],
     36     'content': df['content'][0]
     37 }

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py:397, in read_sql_query(sql, con, index_col, coerce_float, params, parse_dates, chunksize, dtype)
    339 """
    340 Read SQL query into a DataFrame.
    341 
   (...)
    394 parameter will be converted to UTC.
    395 """
    396 pandas_sql = pandasSQL_builder(con)
--> 397 return pandas_sql.read_query(
    398     sql,
    399     index_col=index_col,
    400     params=params,
    401     coerce_float=coerce_float,
    402     parse_dates=parse_dates,
    403     chunksize=chunksize,
    404     dtype=dtype,
    405 )

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py:1560, in SQLDatabase.read_query(self, sql, index_col, coerce_float, parse_dates, params, chunksize, dtype)
   1512 """
   1513 Read SQL query into a DataFrame.
   1514 
   (...)
   1556 
   1557 """
   1558 args = _convert_params(sql, params)
-> 1560 result = self.execute(*args)
   1561 columns = result.keys()
   1563 if chunksize is not None:

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/pandas/io/sql.py:1405, in SQLDatabase.execute(self, *args, **kwargs)
   1403 def execute(self, *args, **kwargs):
   1404     """Simple passthrough to SQLAlchemy connectable"""
-> 1405     return self.connectable.execution_options().execute(*args, **kwargs)

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1365, in Connection.execute(self, statement, *multiparams, **params)
   1356 if isinstance(statement, util.string_types):
   1357     util.warn_deprecated_20(
   1358         "Passing a string to Connection.execute() is "
   1359         "deprecated and will be removed in version 2.0.  Use the "
   (...)
   1362         "driver-level SQL string."
   1363     )
-> 1365     return self._exec_driver_sql(
   1366         statement,
   1367         multiparams,
   1368         params,
   1369         _EMPTY_EXECUTION_OPTS,
   1370         future=False,
   1371     )
   1373 try:
   1374     meth = statement._execute_on_connection

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1669, in Connection._exec_driver_sql(self, statement, multiparams, params, execution_options, future)
   1659         (
   1660             statement,
   1661             distilled_params,
   (...)
   1665             statement, distilled_parameters, execution_options
   1666         )
   1668 dialect = self.dialect
-> 1669 ret = self._execute_context(
   1670     dialect,
   1671     dialect.execution_ctx_cls._init_statement,
   1672     statement,
   1673     distilled_parameters,
   1674     execution_options,
   1675     statement,
   1676     distilled_parameters,
   1677 )
   1679 if not future:
   1680     if self._has_events or self.engine._has_events:

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1943, in Connection._execute_context(self, dialect, constructor, statement, parameters, execution_options, *args, **kw)
   1940             branched.close()
   1942 except BaseException as e:
-> 1943     self._handle_dbapi_exception(
   1944         e, statement, parameters, cursor, context
   1945     )
   1947 return result

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:2128, in Connection._handle_dbapi_exception(self, e, statement, parameters, cursor, context)
   2124         util.raise_(
   2125             sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2126         )
   2127     else:
-> 2128         util.raise_(exc_info[1], with_traceback=exc_info[2])
   2130 finally:
   2131     del self._reentrant_error

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py:211, in raise_(***failed resolving arguments***)
    208     exception.__cause__ = replace_context
    210 try:
--> 211     raise exception
    212 finally:
    213     # credit to
    214     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    215     # as the __traceback__ object creates a cycle
    216     del exception, replace_context, from_, with_traceback

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1900, in Connection._execute_context(self, dialect, constructor, statement, parameters, execution_options, *args, **kw)
   1898                 break
   1899     if not evt_handled:
-> 1900         self.dialect.do_execute(
   1901             cursor, statement, parameters, context
   1902         )
   1904 if self._has_events or self.engine._has_events:
   1905     self.dispatch.after_cursor_execute(
   1906         self,
   1907         cursor,
   (...)
   1911         context.executemany,
   1912     )

File ~/projects/twitter_scraper/venv/lib/python3.8/site-packages/sqlalchemy/engine/default.py:736, in DefaultDialect.do_execute(self, cursor, statement, parameters, context)
    735 def do_execute(self, cursor, statement, parameters, context=None):
--> 736     cursor.execute(statement, parameters)

File /usr/lib/python3.8/encodings/utf_8.py:15, in decode(input, errors)
     11 ### Codec APIs
     13 encode = codecs.utf_8_encode
---> 15 def decode(input, errors='strict'):
     16     return codecs.utf_8_decode(input, errors, True)
     18 class IncrementalEncoder(codecs.IncrementalEncoder):

KeyboardInterrupt: 

In [ ]: