From 5de622a2bfbb369a690fa0ad4422e2991b3af4e2 Mon Sep 17 00:00:00 2001
From: Brian Bjarke Jensen <bbj@skov.dk>
Date: Fri, 1 Mar 2024 13:25:08 +0100
Subject: [PATCH] added script to prepare csv source file

---
 image_download/prepare_csv.py | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 image_download/prepare_csv.py

diff --git a/image_download/prepare_csv.py b/image_download/prepare_csv.py
new file mode 100644
index 0000000..f870002
--- /dev/null
+++ b/image_download/prepare_csv.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+
+
+if __name__ == '__main__':
+    # prepare csv path
+    input_path = Path(__file__).parent / 'images_mm.csv'
+    assert input_path.exists()
+    # load in csv
+    df = pd.read_csv(
+        input_path,
+        delimiter=';',
+        header=None,
+        names=['name', 'url'],
+    )
+    # remove last letter from names
+    df['name'] = df['name'].str[:-1]
+    # remove rows with duplicate names
+    df.drop_duplicates(subset='name', inplace=True)
+    # reset index to look more nice
+    df.reset_index(drop=True, inplace=True)
+    # remove query from urls
+    pattern = r'(\?img_.+)'
+    df['url'] = df['url'].str.replace(
+        pat=pattern,
+        repl='',
+        regex=True,
+    )
+    # save df to file
+    output_path = input_path.parent / 'sources.csv'
+    df.to_csv(output_path, index=False)