From 21ecfb8990efbd082b95b59b036263e209aedd38 Mon Sep 17 00:00:00 2001
From: simplypower-bbj <bbj@simplypower.dk>
Date: Mon, 9 Jan 2023 19:13:51 +0100
Subject: [PATCH] removed unused script

---
 code/main.py | 76 ----------------------------------------------------
 1 file changed, 76 deletions(-)
 delete mode 100644 code/main.py

diff --git a/code/main.py b/code/main.py
deleted file mode 100644
index 2db0ca8..0000000
--- a/code/main.py
+++ /dev/null
@@ -1,76 +0,0 @@
-# public packages
-import snscrape.modules.twitter as sntwitter
-from tqdm import tqdm
-import time
-# database-related
-from database import Data_table, insert_list, get_all
-from dotenv import load_dotenv
-# own code
-from clean_data import clean_content_df
-from translate import translate
-
-def run_snscrape():
-    # prepare lists to hold information
-    username_list = list()
-    timestamp_list = list()
-    content_list = list()
-    # begin scraping
-    query = '(Odense OR odense OR #odense OR #Odense) until:2022-12-17 since:2017-01-01 -filter:replies'
-    for i, tweet in enumerate(sntwitter.TwitterSearchScraper(query).get_items()):
-        # store information in lists
-        username_list.append(tweet.user.username)
-        timestamp_list.append(tweet.date)
-        content_list.append(tweet.content)
-        # when 100 tweets have been collected
-        if len(content_list) == 100:
-            # send data to database
-            insert_list(username_list, timestamp_list, content_list)
-            # empty lists
-            username_list = list()
-            timestamp_list = list()
-            content_list = list()
-        # check in on progress
-        if i % 100 == 0:
-            print(f'found {i} tweets')
-
-def translate_content():
-    # load all content from database
-    print('loading data from database...')
-    df = get_all()
-    # remove already translated content
-    df = df[df['content_en'].isna()]
-    # clean content
-    print('cleaning content...')
-    content_clean_list, id_list = clean_content_df(df)
-    # translate content
-    print('translating content...')
-    with Data_table() as dt:
-        for id, content in tqdm(zip(id_list, content_clean_list), ascii=True, total=len(id_list)):
-            # translate content
-            try:
-                content_trans = translate(content)
-            except Exception:
-                print(f'error when translating content with id={id}')
-                continue
-            # transfer to database
-            try:
-                val_dict = {
-                    'id': id,
-                    'content_en': content_trans
-                }
-                query = (
-                    f'UPDATE {dt._name} '
-                    ' SET content_en = (%(content_en)s) '
-                    'WHERE id = (%(id)s) '
-                )
-                dt.execute(query, val_dict)
-            except Exception:
-                print(f'error when sending translated content to database: id={id}')
-            # take a break to avoid being banned by Google
-            time.sleep(2)
-
-
-if __name__ == '__main__':
-    load_dotenv()
-    # run_snscrape()
-    translate_content()
\ No newline at end of file