316 lines
12 KiB
Plaintext
316 lines
12 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>id</th>\n",
|
||
" <th>username</th>\n",
|
||
" <th>time</th>\n",
|
||
" <th>content</th>\n",
|
||
" <th>content_en</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td>190</td>\n",
|
||
" <td>@WorkWithUsDK</td>\n",
|
||
" <td>2022-12-17 08:28:03-08:00</td>\n",
|
||
" <td>Odense Universitets Hospital søger #Afdelingsc...</td>\n",
|
||
" <td>Odense University Hospital Seeking #Department...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>1</th>\n",
|
||
" <td>157</td>\n",
|
||
" <td>@westblick</td>\n",
|
||
" <td>2022-12-17 05:05:03-08:00</td>\n",
|
||
" <td>\"Odense Dänemark, Seaside\" Fotografie als Post...</td>\n",
|
||
" <td>\"Odense Dänemark, Seaside\" photography ALS Pos...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2</th>\n",
|
||
" <td>223</td>\n",
|
||
" <td>EryilmazFa10</td>\n",
|
||
" <td>2022-12-16 14:20:26-08:00</td>\n",
|
||
" <td>Danimarka’nın Odense takımı bu yaz müthiş bir ...</td>\n",
|
||
" <td>Denmark's Odense team made a great scouting tr...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3</th>\n",
|
||
" <td>224</td>\n",
|
||
" <td>trans_rumor</td>\n",
|
||
" <td>2022-12-16 13:56:35-08:00</td>\n",
|
||
" <td>Highlights: SønderjyskE – Odense https://t.co/...</td>\n",
|
||
" <td>Highlights: SønderjyskE - Odense https://t.co/...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>4</th>\n",
|
||
" <td>225</td>\n",
|
||
" <td>sakak</td>\n",
|
||
" <td>2022-12-16 13:39:40-08:00</td>\n",
|
||
" <td>“Meta Platforms Inc has halted construction of...</td>\n",
|
||
" <td>\"meta platforms inc has halted construction of...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>...</th>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" <td>...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>154110</th>\n",
|
||
" <td>154518</td>\n",
|
||
" <td>JobiOdense</td>\n",
|
||
" <td>2017-01-02 07:31:03-08:00</td>\n",
|
||
" <td>Køkkensælger til Danmarks nye køkkenkoncept ht...</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>154111</th>\n",
|
||
" <td>154519</td>\n",
|
||
" <td>GunnarBjorns</td>\n",
|
||
" <td>2017-01-02 07:22:27-08:00</td>\n",
|
||
" <td>watching Sunderland vs Liverpool at The Old Ir...</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>154112</th>\n",
|
||
" <td>154520</td>\n",
|
||
" <td>NordicDate</td>\n",
|
||
" <td>2017-01-02 07:20:09-08:00</td>\n",
|
||
" <td>Danmarks størreste #sex #dating portal:\\n\\nhtt...</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>154113</th>\n",
|
||
" <td>154521</td>\n",
|
||
" <td>DBAfeed</td>\n",
|
||
" <td>2017-01-02 07:18:05-08:00</td>\n",
|
||
" <td>Elguitar, Morgan Stratocaster kopi, Sælger de...</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>154114</th>\n",
|
||
" <td>154522</td>\n",
|
||
" <td>TRungCardio</td>\n",
|
||
" <td>2017-01-02 07:10:28-08:00</td>\n",
|
||
" <td>Bevar mulighederne for børn i Odense... Og lad...</td>\n",
|
||
" <td>None</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"<p>154115 rows × 5 columns</p>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" id username time \\\n",
|
||
"0 190 @WorkWithUsDK 2022-12-17 08:28:03-08:00 \n",
|
||
"1 157 @westblick 2022-12-17 05:05:03-08:00 \n",
|
||
"2 223 EryilmazFa10 2022-12-16 14:20:26-08:00 \n",
|
||
"3 224 trans_rumor 2022-12-16 13:56:35-08:00 \n",
|
||
"4 225 sakak 2022-12-16 13:39:40-08:00 \n",
|
||
"... ... ... ... \n",
|
||
"154110 154518 JobiOdense 2017-01-02 07:31:03-08:00 \n",
|
||
"154111 154519 GunnarBjorns 2017-01-02 07:22:27-08:00 \n",
|
||
"154112 154520 NordicDate 2017-01-02 07:20:09-08:00 \n",
|
||
"154113 154521 DBAfeed 2017-01-02 07:18:05-08:00 \n",
|
||
"154114 154522 TRungCardio 2017-01-02 07:10:28-08:00 \n",
|
||
"\n",
|
||
" content \\\n",
|
||
"0 Odense Universitets Hospital søger #Afdelingsc... \n",
|
||
"1 \"Odense Dänemark, Seaside\" Fotografie als Post... \n",
|
||
"2 Danimarka’nın Odense takımı bu yaz müthiş bir ... \n",
|
||
"3 Highlights: SønderjyskE – Odense https://t.co/... \n",
|
||
"4 “Meta Platforms Inc has halted construction of... \n",
|
||
"... ... \n",
|
||
"154110 Køkkensælger til Danmarks nye køkkenkoncept ht... \n",
|
||
"154111 watching Sunderland vs Liverpool at The Old Ir... \n",
|
||
"154112 Danmarks størreste #sex #dating portal:\\n\\nhtt... \n",
|
||
"154113 Elguitar, Morgan Stratocaster kopi, Sælger de... \n",
|
||
"154114 Bevar mulighederne for børn i Odense... Og lad... \n",
|
||
"\n",
|
||
" content_en \n",
|
||
"0 Odense University Hospital Seeking #Department... \n",
|
||
"1 \"Odense Dänemark, Seaside\" photography ALS Pos... \n",
|
||
"2 Denmark's Odense team made a great scouting tr... \n",
|
||
"3 Highlights: SønderjyskE - Odense https://t.co/... \n",
|
||
"4 \"meta platforms inc has halted construction of... \n",
|
||
"... ... \n",
|
||
"154110 None \n",
|
||
"154111 None \n",
|
||
"154112 None \n",
|
||
"154113 None \n",
|
||
"154114 None \n",
|
||
"\n",
|
||
"[154115 rows x 5 columns]"
|
||
]
|
||
},
|
||
"execution_count": 1,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"from database import get_all\n",
|
||
"\n",
|
||
"df = get_all()\n",
|
||
"\n",
|
||
"df"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"0 odense universitets hospital søger #afdelingsc...\n",
|
||
"1 \"odense dänemark, seaside\" fotografie als post...\n",
|
||
"2 danimarka'nın odense takımı bu yaz müthiş bir ...\n",
|
||
"3 highlights: sønderjyske – odense https://t.co/...\n",
|
||
"4 \"meta platforms inc has halted construction of...\n",
|
||
"5 sponsoreret: verdensmestre, europamestre og st...\n",
|
||
"6 sponsoreret: verdensmestre, europamestre og st...\n",
|
||
"7 meta just halted a major data centre project t...\n",
|
||
"8 ende #eishockey herren #den #metalligaen\\n5:3 ...\n",
|
||
"9 ende #eishockey herren #den #metalligaen\\n5:3 ...\n",
|
||
"Name: content, dtype: object"
|
||
]
|
||
},
|
||
"execution_count": 2,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"from clean_data import clean_content_df\n",
|
||
"\n",
|
||
"content_clean, id_list = clean_content_df(df.iloc[:10, :])\n",
|
||
"\n",
|
||
"content_clean"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"['Odense University Hospital Seeking #Department Manager in #Odense. Read more at http://workwithhus.dk/open-soeger-visionaer-og-erfaren-lead-with-policy-taeft… #jobdk #jobiodense ',\n",
|
||
" '\"Odense Dänemark, Seaside\" photography ALS Poster und Kstorpel von Robert West Bestellen. -http://artflakes.com https://artflakes.com/de/products/odense-danemark-seaside ... via @artflakes #odense #Denmark #dänemark #seaside #panorama #artflakes #bigprint #artwork #fuji #photography #landscapephotography ',\n",
|
||
" \"Denmark's Odense team made a great scouting transfer this summer and bought Franco Tonga for free. The man was invited to the Italian national team. https://t.co/fdxmsbuti4 https://t.co/vbcyr871FF \",\n",
|
||
" 'Highlights: SønderjyskE - Odense https://t.co/b4ounlj6on #highlights #hockeymagasinetdk ',\n",
|
||
" '\"meta platforms inc has halted construction of two data centres in odense, denmark, and will instead focus on a new type of data centre used for artificial intelligence (ai), a spokesperson said on thursday.\" https://t.co/gz09vii8hx',\n",
|
||
" 'Sponsored: World champions, European champions and large Danish medalists hit Odense for a prestigious event🚴 #Odense #fyn #cykling https://t.co/u0vpdlolvp ',\n",
|
||
" 'Sponsored: World champions, European champions and large Danish medalists hit Odense for a prestigious event #Odense #fyn #cykling ',\n",
|
||
" 'meta just halted a major data centre project to focus on ai - https://t.co/scksqvfuyk\\nmeta has axed a major data center expansion in odense, denmark, laying off the primary contractor on two facilities in the country. according to reporting by data centre dynamics (opens... https://t.co/eslgjl4wtk',\n",
|
||
" 'end #eishockey the gentleman #de #metalliga 5: 3 #AALBORG VS ESBJERG 2: 9 Rodovre vs #rungsted 2: 6 Sonderjyske vs #ODENSE ',\n",
|
||
" 'end #eishockey the gentleman #de #metalliga 5: 3 #AALBORG VS ESBJERG 2: 9 Rodovre vs #rungsted 2: 6 Sonderjyske vs #ODENSE ',\n",
|
||
" 'ERHR Recruitment Seeking #Quality Manager in #Odense. Read more at https://t.co/sbm3ittbaf #jobdk #jobiodense ']"
|
||
]
|
||
},
|
||
"execution_count": 3,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"from translate import translate\n",
|
||
"\n",
|
||
"# translate\n",
|
||
"translated_list = content_clean.apply(translate).to_list()\n",
|
||
"\n",
|
||
"translated_list"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"[190, 157, 223, 224, 225, 226, 1, 227, 2, 228, 229]"
|
||
]
|
||
},
|
||
"execution_count": 5,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"# try to send translation to database\n",
|
||
"from database import insert_translated_list\n",
|
||
"\n",
|
||
"insert_translated_list(translated_list, id_list)\n",
|
||
"\n",
|
||
"id_list"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "venv",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.8.10"
|
||
},
|
||
"orig_nbformat": 4,
|
||
"vscode": {
|
||
"interpreter": {
|
||
"hash": "57b41ea9bda188de64800ef31c5cfb4fcfca577257038becb66e4fa0a5ee97a6"
|
||
}
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 2
|
||
}
|