From 2cb772c074c0219f3b191cd7ab1042ae67df897b Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 29 Jan 2024 16:06:26 -0500 Subject: [PATCH 1/4] Add script to automatically delete old log files --- jwql/website/apps/jwql/clean_old_log_files.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 jwql/website/apps/jwql/clean_old_log_files.py diff --git a/jwql/website/apps/jwql/clean_old_log_files.py b/jwql/website/apps/jwql/clean_old_log_files.py new file mode 100644 index 000000000..8d0de248c --- /dev/null +++ b/jwql/website/apps/jwql/clean_old_log_files.py @@ -0,0 +1,83 @@ +#! /usr/bin/env python + +"""Clean old log files from the collection of log files + +Authors +------- + + - Bryan Hilbert + +Use +--- + + To delete log files that are older than some threshold age: + :: + + > python clean_old_log_files.py --time_limit 7 # days + +""" +import argparse +from datetime import datetime, timedelta +import os +import socket + +from jwql.utils.utils import get_config + +HOSTNAME = socket.gethostname() +configs = get_config() +LOG_BASE_DIR = configs['log_dir'] + + +def define_options(): + """Create parser to take the time limit. + + Returns + ------- + parser : argparse.ArgumentParser + Parser containing time limit + """ + usage = 'clean_old_log_files.py -t 14' + parser = argparse.ArgumentParser(usage=usage) + parser.add_argument('-t', '--time_limit', type=int, default=14, + help='Time limit in days. Log files older than this will be deleted.') + return parser + +def run(time_limit=timedelta(days=14)): + """Look through log directories and delete log files that are older than ``time_limit``. + Have time_limit default to be 14 days. + + Inputs + ------ + time_limit : datetime.timdelta + Files older than this time limit will be deleted + """ + now = datetime.now() + + if 'pljwql' in HOSTNAME: + subdir = 'ops' + elif 'tljwql' in HOSTNAME: + subdir = 'test' + else: + # This should cover the dev server as well as local machines + subdir = 'dev' + + log_dir = os.path.join(LOG_BASE_DIR, subdir) + for logtype in os.scandir(log_dir): + if logtype.is_dir(): + for item in os.scandir(logtype): + # We only try to delete log files produced by the machine one which + # this script is running. e.g. + if HOSTNAME in item.name and item.name[-4:] == '.log': + stat_result = item.stat() + last_modified_time = datetime.fromtimestamp(stat_result.st_mtime) + age = now - last_modified_time + if age > time_limit: + #os.remove(item.name) + full_path = os.path.join(log_dir, logtype, item) + print(f'DELETING {full_path}') + print('Once initial testing is complete, replace the print statement above with the os.remove line above it.') + +if __name__ == '__main__': + parser = define_options() + args = parser.parse_args() + run(timedelta(days=args.time_limit)) From c9289df096d748b330e8615e9cb5dae8de750a54 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 29 Jan 2024 16:12:49 -0500 Subject: [PATCH 2/4] PEP8 --- jwql/website/apps/jwql/clean_old_log_files.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/clean_old_log_files.py b/jwql/website/apps/jwql/clean_old_log_files.py index 8d0de248c..62f910ae2 100644 --- a/jwql/website/apps/jwql/clean_old_log_files.py +++ b/jwql/website/apps/jwql/clean_old_log_files.py @@ -42,6 +42,7 @@ def define_options(): help='Time limit in days. Log files older than this will be deleted.') return parser + def run(time_limit=timedelta(days=14)): """Look through log directories and delete log files that are older than ``time_limit``. Have time_limit default to be 14 days. @@ -65,7 +66,7 @@ def run(time_limit=timedelta(days=14)): for logtype in os.scandir(log_dir): if logtype.is_dir(): for item in os.scandir(logtype): - # We only try to delete log files produced by the machine one which + # We only try to delete log files produced by the machine on which # this script is running. e.g. if HOSTNAME in item.name and item.name[-4:] == '.log': stat_result = item.stat() @@ -77,6 +78,7 @@ def run(time_limit=timedelta(days=14)): print(f'DELETING {full_path}') print('Once initial testing is complete, replace the print statement above with the os.remove line above it.') + if __name__ == '__main__': parser = define_options() args = parser.parse_args() From 1305244ad71d2a1436023925e75de6bf3084a6ad Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 29 Jan 2024 16:14:08 -0500 Subject: [PATCH 3/4] Fix doc string --- jwql/website/apps/jwql/clean_old_log_files.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jwql/website/apps/jwql/clean_old_log_files.py b/jwql/website/apps/jwql/clean_old_log_files.py index 62f910ae2..ae8e3ba55 100644 --- a/jwql/website/apps/jwql/clean_old_log_files.py +++ b/jwql/website/apps/jwql/clean_old_log_files.py @@ -67,7 +67,8 @@ def run(time_limit=timedelta(days=14)): if logtype.is_dir(): for item in os.scandir(logtype): # We only try to delete log files produced by the machine on which - # this script is running. e.g. + # this script is running. e.g. log files produced by the test server + # can only be deleted by running this script on the test server. if HOSTNAME in item.name and item.name[-4:] == '.log': stat_result = item.stat() last_modified_time = datetime.fromtimestamp(stat_result.st_mtime) From c2f3055f74dfb72db9575829ad0dfe1e0394700f Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 30 Jan 2024 10:23:14 -0500 Subject: [PATCH 4/4] Uncomment os.remove line --- jwql/website/apps/jwql/clean_old_log_files.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jwql/website/apps/jwql/clean_old_log_files.py b/jwql/website/apps/jwql/clean_old_log_files.py index ae8e3ba55..dd0a6b95c 100644 --- a/jwql/website/apps/jwql/clean_old_log_files.py +++ b/jwql/website/apps/jwql/clean_old_log_files.py @@ -74,10 +74,8 @@ def run(time_limit=timedelta(days=14)): last_modified_time = datetime.fromtimestamp(stat_result.st_mtime) age = now - last_modified_time if age > time_limit: - #os.remove(item.name) full_path = os.path.join(log_dir, logtype, item) - print(f'DELETING {full_path}') - print('Once initial testing is complete, replace the print statement above with the os.remove line above it.') + os.remove(full_path) if __name__ == '__main__':