From 652fe9af3a785a256415ad60480735f11585c807 Mon Sep 17 00:00:00 2001 From: brb Date: Wed, 12 Jul 2023 13:17:12 +0200 Subject: [PATCH 01/14] added checking for environment variables --- code/database.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/code/database.py b/code/database.py index 2123f59..800deb1 100644 --- a/code/database.py +++ b/code/database.py @@ -3,6 +3,7 @@ from dotenv import load_dotenv from configparser import ConfigParser from pathlib import Path import logging +import os # load default values @@ -21,6 +22,20 @@ def connect( """ Connect to MongoDB database and return collection. """ + # load environment variables + load_dotenv() + ip_addr = os.getenv( + 'DB_IP_ADDRESS', + default=DB_IP_ADDRESS + ) + db_name = os.getenv( + 'DB_NAME', + default=DB_NAME + ) + collection_name = os.getenv( + 'DB_COLLECTION_NAME', + default=DB_COLLECTION_NAME + ) # connect to database client: MongoClient = MongoClient(ip_addr) db = client[db_name] From ae26182e2354b224118735d6312c0a668455274d Mon Sep 17 00:00:00 2001 From: brb Date: Wed, 12 Jul 2023 13:26:12 +0200 Subject: [PATCH 02/14] updated default values --- code/defaults.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/code/defaults.ini b/code/defaults.ini index 3d0ba6b..7f9cdb1 100644 --- a/code/defaults.ini +++ b/code/defaults.ini @@ -1,8 +1,8 @@ [main] -logger_level=info +logger_level=debug user_id=0 replicates=3 -trigger_interval_seconds=60 +trigger_interval_seconds=3600 [database] db_ip_address=192.168.1.2 From a456ccc5365855beace24de65a06437352c53cc3 Mon Sep 17 00:00:00 2001 From: brb Date: Wed, 12 Jul 2023 13:35:40 +0200 Subject: [PATCH 03/14] updated to check for env var USER_ID --- code/initialise_app.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/code/initialise_app.py b/code/initialise_app.py index 2307209..a22b2b8 100644 --- a/code/initialise_app.py +++ b/code/initialise_app.py @@ -20,13 +20,15 @@ def initialise_app( discord_service_name: str = DISCORD_SERVICE_NAME, discord_webhook_url: str = DISCORD_WEBHOOK_URL, discord_logger_level: str = DISCORD_LOGGER_LEVEL -): +) -> int: """ Convienience function that ensures eveything is ready before running the main loop. """ # load environment variables load_dotenv() + user_id = int(os.getenv('USER_ID', default=None)) + assert user_id is not None, 'environment variable USER_ID must be specified' logger_level = os.getenv( 'LOGGER_LEVEL', default=LOGGER_LEVEL @@ -58,8 +60,10 @@ def initialise_app( level = getattr(logging, discord_logger_level.upper()) discord_handler.setLevel(level=level) logger.addHandler(discord_handler) + logging.debug('finished') + return user_id if __name__ == '__main__': - initialise_app() + user_id = initialise_app() logging.warning('test message') From caeace7907d2aec4338b49625d8ddac71202ddce Mon Sep 17 00:00:00 2001 From: brb Date: Wed, 12 Jul 2023 13:46:01 +0200 Subject: [PATCH 04/14] updated to check for env var USER_ID --- code/initialise_app.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/code/initialise_app.py b/code/initialise_app.py index a22b2b8..ba0a63b 100644 --- a/code/initialise_app.py +++ b/code/initialise_app.py @@ -20,15 +20,14 @@ def initialise_app( discord_service_name: str = DISCORD_SERVICE_NAME, discord_webhook_url: str = DISCORD_WEBHOOK_URL, discord_logger_level: str = DISCORD_LOGGER_LEVEL -) -> int: +) -> None: """ Convienience function that ensures eveything is ready before running the main loop. """ # load environment variables load_dotenv() - user_id = int(os.getenv('USER_ID', default=None)) - assert user_id is not None, 'environment variable USER_ID must be specified' + assert 'USER_ID' in os.environ, 'environment variable USER_ID must be specified' logger_level = os.getenv( 'LOGGER_LEVEL', default=LOGGER_LEVEL @@ -61,7 +60,6 @@ def initialise_app( discord_handler.setLevel(level=level) logger.addHandler(discord_handler) logging.debug('finished') - return user_id if __name__ == '__main__': From ecfddfc69e59603a85f810014932a4d65caf64d4 Mon Sep 17 00:00:00 2001 From: brb Date: Wed, 12 Jul 2023 13:47:37 +0200 Subject: [PATCH 05/14] added data_queue flush interval --- code/defaults.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/code/defaults.ini b/code/defaults.ini index 7f9cdb1..afc71aa 100644 --- a/code/defaults.ini +++ b/code/defaults.ini @@ -3,6 +3,7 @@ logger_level=debug user_id=0 replicates=3 trigger_interval_seconds=3600 +data_flush_interval_seconds=600 [database] db_ip_address=192.168.1.2 From e3fadbc22cc69887af74971f17c921446479ddd4 Mon Sep 17 00:00:00 2001 From: brb Date: Wed, 12 Jul 2023 13:47:56 +0200 Subject: [PATCH 06/14] added data_queue flush interval --- code/defaults.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/defaults.ini b/code/defaults.ini index afc71aa..c4677b8 100644 --- a/code/defaults.ini +++ b/code/defaults.ini @@ -3,7 +3,7 @@ logger_level=debug user_id=0 replicates=3 trigger_interval_seconds=3600 -data_flush_interval_seconds=600 +data_queue_flush_interval_seconds=600 [database] db_ip_address=192.168.1.2 From 12ccd1e586c3319fc31923a62b654bb423d76300 Mon Sep 17 00:00:00 2001 From: brb Date: Wed, 12 Jul 2023 13:59:53 +0200 Subject: [PATCH 07/14] added extra data flushing event with separate trigger time --- code/main.py | 119 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 77 insertions(+), 42 deletions(-) diff --git a/code/main.py b/code/main.py index 5b84253..0ef9e24 100644 --- a/code/main.py +++ b/code/main.py @@ -14,45 +14,28 @@ from database import connect config = ConfigParser() config.read(Path(__file__).parent / 'defaults.ini') REPLICATES = int(config.get('main', 'replicates')) -USER_ID = int(config.get('main', 'user_id')) TRIGGER_INTERVAL_SECONDS = float( config.get( 'main', 'trigger_interval_seconds' ) ) +DATA_QUEUE_FLUSH_INTERVAL_SECONDS = float( + config.get( + 'main', + 'data_queue_flush_interval_seconds' + ) +) -def event( - replicates: int = REPLICATES, - user_id: int = USER_ID -): - logging.debug('started event') - # load environment variables - replicates = int(os.getenv('REPLICATES', default=REPLICATES)) - user_id = int(os.getenv('USER_ID', default=USER_ID)) - # setup data queue - data_queue: Queue = Queue() - # run event - for rep_num in range(replicates): - logging.debug(f'running replicate {rep_num+1} of {replicates}') - # do measurement - try: - data = measure_bandwidth() - except Exception as e: - logging.error(f'failed to measure bandwidth: {e}') - continue - # add data to queue - data_queue.put(data) - # polite pause - time.sleep(1) - # check if queue should be flushed - queue_size = data_queue.qsize() - if queue_size == 0: - logging.debug('queue empty, continuing') - return - # flush queue to database +def flush_data( + data_queue: Queue, + user_id: int +) -> None: + # connect to database db = connect() + # flush queue + queue_size = data_queue.qsize() for i in range(queue_size): # get data data = data_queue.get() @@ -78,26 +61,78 @@ def event( logging.debug('finished') +def event( + data_queue: Queue, + user_id: int, + replicates: int = REPLICATES +) -> None: + # run event + for rep_num in range(replicates): + logging.debug(f'running replicate {rep_num+1} of {replicates}') + # do measurement + try: + data = measure_bandwidth() + except Exception as e: + logging.error(f'failed to measure bandwidth: {e}') + continue + # add data to queue + data_queue.put(data) + # polite pause + time.sleep(1) + # flush queue + if data_queue.qsize() > 0: + flush_data( + data_queue=data_queue, + user_id=user_id + ) + logging.debug('finished') + + if __name__ == '__main__': + # setup initialise_app() + user_id = int(os.getenv('USER_ID', default=0)) + replicates = int(os.getenv('REPLICATES', default=REPLICATES)) + trigger_interval = float( + os.getenv( + 'TRIGGER_INTERVAL_SECONDS', + default=TRIGGER_INTERVAL_SECONDS + ) + ) + data_flush_interval = float( + os.getenv( + 'DATA_QUEUE_FLUSH_INTERVAL_SECONDS', + default=DATA_QUEUE_FLUSH_INTERVAL_SECONDS + ) + ) + data_queue: Queue = Queue() + # start main loop trigger_time = time.time() + data_flush_time = trigger_time + data_flush_interval while True: + # measure event if time.time() >= trigger_time: logging.debug('triggered event') # set new trigger time - trigger_interval = float( - os.getenv( - 'TRIGGER_INTERVAL_SECONDS', - default=TRIGGER_INTERVAL_SECONDS - ) - ) trigger_time += trigger_interval # run event - try: - event() - except Exception as e: - logging.error(e) - else: - logging.info('finished event') + event( + data_queue=data_queue, + user_id=user_id, + replicates=replicates + ) + # extra data flush + if ( + time.time() >= data_flush_time and + data_queue.qsize() > 0 + ): + logging.debug('triggered extra data flush') + # set new trigger time + data_flush_time += data_flush_interval + # flush data + flush_data( + data_queue=data_queue, + user_id=user_id + ) # polite pause time.sleep(1) From d5bcf04e611d057a8bf66417dcc6ab097096a796 Mon Sep 17 00:00:00 2001 From: brb Date: Wed, 12 Jul 2023 14:01:06 +0200 Subject: [PATCH 08/14] updated readme --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 0ceeb97..60977f1 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,21 @@ # bandwidth_probing Service that runs in a container and measures network bandwidth at a specified interval. The results are sent to a MongoDB database. Logs are sent to discord channel. + +## Environment Variables + +```python +LOGGER_LEVEL: Literal['debug', 'info', 'warning', 'error', 'critical'] = 'debug' +USER_ID: int = 0 # id number 0 is reserved for testing +REPLICATES: int = 3 # number of replicates to be measured per event +TRIGGER_INTERVAL_SECONDS: float = 3600 # seconds between events +DATA_QUEUE_FLUSH_INTERVAL_SECONDS: float = 600 # seconds between attempts to push data stuck in data queue + +DB_IP_ADDRESS: str = '192.168.1.2' +DB_NAME: str = 'bandwidth_probing' +DB_COLLECTION_NAME: str = 'data' + +DISCORD_SERVICE_NAME: str = 'bandwidth_probing +DISCORD_WEBHOOK_URL: str = 'https://discord.com/api/webhooks/1127970367047225354/wNkoRex4OncMw11OCJg6atR_xgHS2VhLrku4jYIUw81Kr4sutdD3Tt-XAltG0UQy14rZ' +DISCORD_LOGGER_LEVEL: Literal['debug', 'info', 'warning', 'error', 'critical'] = 'warning' +``` From 387986e2d843ad351c10ae26b7e6bcfd16ffb9d9 Mon Sep 17 00:00:00 2001 From: brb Date: Wed, 12 Jul 2023 14:03:41 +0200 Subject: [PATCH 09/14] flake8 compliant --- code/database.py | 2 +- code/initialise_app.py | 6 +++++- code/main.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/code/database.py b/code/database.py index 800deb1..5eb535f 100644 --- a/code/database.py +++ b/code/database.py @@ -25,7 +25,7 @@ def connect( # load environment variables load_dotenv() ip_addr = os.getenv( - 'DB_IP_ADDRESS', + 'DB_IP_ADDRESS', default=DB_IP_ADDRESS ) db_name = os.getenv( diff --git a/code/initialise_app.py b/code/initialise_app.py index ba0a63b..973ae74 100644 --- a/code/initialise_app.py +++ b/code/initialise_app.py @@ -27,7 +27,11 @@ def initialise_app( """ # load environment variables load_dotenv() - assert 'USER_ID' in os.environ, 'environment variable USER_ID must be specified' + assert ( + 'USER_ID' in os.environ + ), ( + 'environment variable USER_ID must be specified' + ) logger_level = os.getenv( 'LOGGER_LEVEL', default=LOGGER_LEVEL diff --git a/code/main.py b/code/main.py index 0ef9e24..ee07606 100644 --- a/code/main.py +++ b/code/main.py @@ -123,7 +123,7 @@ if __name__ == '__main__': ) # extra data flush if ( - time.time() >= data_flush_time and + time.time() >= data_flush_time and data_queue.qsize() > 0 ): logging.debug('triggered extra data flush') From 63975169c0aefbc8e4a23d97359f95db0fab5bd1 Mon Sep 17 00:00:00 2001 From: brb Date: Wed, 12 Jul 2023 14:04:10 +0200 Subject: [PATCH 10/14] mypy compliant --- code/initialise_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/initialise_app.py b/code/initialise_app.py index 973ae74..d16a0c8 100644 --- a/code/initialise_app.py +++ b/code/initialise_app.py @@ -67,5 +67,5 @@ def initialise_app( if __name__ == '__main__': - user_id = initialise_app() + initialise_app() logging.warning('test message') From 238188f4a52b115353c62d2feef32736a4cb8d12 Mon Sep 17 00:00:00 2001 From: brb Date: Wed, 12 Jul 2023 14:08:37 +0200 Subject: [PATCH 11/14] introduced flake8 error --- code/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/main.py b/code/main.py index ee07606..74ac07a 100644 --- a/code/main.py +++ b/code/main.py @@ -90,7 +90,7 @@ def event( if __name__ == '__main__': # setup - initialise_app() + initialise_app() user_id = int(os.getenv('USER_ID', default=0)) replicates = int(os.getenv('REPLICATES', default=REPLICATES)) trigger_interval = float( From 7a396a88fb2e2227c869ab9d8f3c8ff1251274ec Mon Sep 17 00:00:00 2001 From: brb Date: Wed, 12 Jul 2023 14:08:51 +0200 Subject: [PATCH 12/14] updated workflow --- .gitea/workflows/default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/default.yaml b/.gitea/workflows/default.yaml index 7151b1f..efb9e33 100644 --- a/.gitea/workflows/default.yaml +++ b/.gitea/workflows/default.yaml @@ -23,6 +23,6 @@ jobs: pip install flake8 mypy pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: PEP8 check - run: flake8 ./code --benchmark --exit-zero + run: flake8 ./code --benchmark - name: Type check run: mypy ./code \ No newline at end of file From d872cfde43cfa642eabca4c45ef6597059bc1f21 Mon Sep 17 00:00:00 2001 From: brb Date: Wed, 12 Jul 2023 14:11:05 +0200 Subject: [PATCH 13/14] flake8 compliant --- code/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/main.py b/code/main.py index 74ac07a..ee07606 100644 --- a/code/main.py +++ b/code/main.py @@ -90,7 +90,7 @@ def event( if __name__ == '__main__': # setup - initialise_app() + initialise_app() user_id = int(os.getenv('USER_ID', default=0)) replicates = int(os.getenv('REPLICATES', default=REPLICATES)) trigger_interval = float( From bc0d2aced72edbde1c227c91ae18e4346d6b8464 Mon Sep 17 00:00:00 2001 From: brb Date: Wed, 12 Jul 2023 14:11:35 +0200 Subject: [PATCH 14/14] updated flake8 outputs --- .gitea/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/release.yaml b/.gitea/workflows/release.yaml index 779f1b9..e515794 100644 --- a/.gitea/workflows/release.yaml +++ b/.gitea/workflows/release.yaml @@ -22,7 +22,7 @@ jobs: pip install flake8 mypy pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: PEP8 check - run: flake8 ./code --benchmark --exit-zero + run: flake8 ./code --benchmark - name: Type check run: mypy ./code publish: