Merge pull request #24 from brandonmcfadd/cleaning-up-the-file-paths

Unifying File Paths

.gitignore

@@ -183,3 +183,4 @@ train_arrivals/json/cta/*.json
 train_arrivals/json/metra/*.json
 train_arrivals/special/last_is_cta_okay_tweets.json
 train_arrivals/json/special-station.json
+train_arrivals/special/last_alert.json

README.md

@@ -46,6 +46,21 @@ The station selected along each line is one where all trains, regardless of dest
     * The Train Tracker API uses Station ID's in integer form, while the Train Tracker Map uses them in float form (I wish I understood why it is this way)
 * 'L' Station codes can be found on the following [site](https://data.cityofchicago.org/Transportation/CTA-System-Information-List-of-L-Stops/8pix-ypme) from the City of Chicago's Data Portal.
 
+## Enviornment File
+* You'll need to create a .env file in your directory to safely store your secrets.
+    * Don't forget to add your .env file to your .gitignore list and never check application secrets, usernames or passwords into a GitHub repo! 
+* The following entries are required in your .env file:
+    * CTA
+        ```
+        TRAIN_API_KEY = 'insert key here'
+        FILE_PATH = 'full file path/cta-reliability'
+        ```
+    * Metra
+        ```
+        METRA_USERNAME = 'metra username'
+        METRA_PASSWORD = 'metra password'
+        ```
+
 ## Running the program
 * Once you have everything [Installed](#Installation) and [Configured](#Configuration) Run the main program `python3 main.py`
 

api/api.py

@@ -50,7 +50,7 @@ def get_date(date_type):
 
 def get_current_username(credentials: HTTPBasicCredentials = Depends(security)):
     """Used to verify Creds"""
-    file = open(file=main_file_path + 'cta-reliability/.tokens',
+    file = open(file=main_file_path + '.tokens',
                 mode='r',
                 encoding='utf-8')
     tokens = json.load(file)
@@ -127,7 +127,7 @@ async def startup():
         "redis://localhost", encoding="utf-8", decode_responses=True)
     # Logging Information
     logger = logging.getLogger("uvicorn.access")
-    log_filename = main_file_path + '/cta-reliability/logs/api-service.log'
+    log_filename = main_file_path + '/logs/api-service.log'
     logging.basicConfig(level=logging.INFO)
     handler = RotatingFileHandler(log_filename, maxBytes=10e6, backupCount=10)
     formatter = logging.Formatter(
@@ -251,7 +251,7 @@ async def return_arrivals_for_date_month_cta_v2(date: str, token: str = Depends(
 async def return_special_station_json(token: str = Depends(get_current_username)):
     """Used to retrieve results"""
     try:
-        json_file = main_file_path + "cta-reliability/train_arrivals/json/special-station.json"
+        json_file = main_file_path + "train_arrivals/json/special-station.json"
         print(json_file)
         results = open(json_file, 'r', encoding="utf-8")
         return Response(content=results.read(), media_type="application/json")
@@ -295,7 +295,7 @@ async def production_upgrade(secret: str, token: str = Depends(get_current_usern
     """Used to trigger upgrade of cta-reliability"""
     try:
         if str(secret) == str(deploy_secret):
-            prod_upgrade = subprocess.run(main_file_path + "cta-reliability/production-upgrade.sh", capture_output=True, check=False)
+            prod_upgrade = subprocess.run(main_file_path + "production-upgrade.sh", capture_output=True, check=False)
             output = prod_upgrade.stdout
         else:
             output = "Invalid Secret"

apps/alerts-to-issues.py

@@ -11,7 +11,7 @@
 github_pat = os.getenv('GITHUB_PAT')
 github_username = os.getenv('GITHUB_USERNAME')
 github_repo = os.getenv('GITHUB_REPO')
-main_file_path = os.getenv('FILE_PATH') + "cta-reliability/"
+main_file_path = os.getenv('FILE_PATH')
 
 def create_github_issue(title,body):
     # GitHub API endpoint for creating issues
@@ -66,5 +66,14 @@ def update_last_alert_number(alert_id):
         json.dump(data, file, indent=4)
     print("JSON file updated successfully.")
 
+# Settings
+file = open(file=main_file_path + 'settings.json',
+            mode='r',
+            encoding='utf-8')
+settings = json.load(file)
+
+# API URL's
+alerts_api_url = settings["alerts-api"]["api-url"]
+
 alerts_to_process = get_alerts()
 process_alerts(alerts_to_process)    

apps/main.py

@@ -27,7 +27,7 @@
 main_file_path = os.getenv('FILE_PATH')
 
 # Logging Information
-LOG_FILENAME = main_file_path + '/cta-reliability/logs/cta-reliability.log'
+LOG_FILENAME = main_file_path + 'logs/cta-reliability.log'
 logging.basicConfig(level=logging.INFO)
 handler = RotatingFileHandler(LOG_FILENAME, maxBytes=10e6, backupCount=10)
 formatter = logging.Formatter(
@@ -136,7 +136,7 @@ def add_train_to_file_api(eta, station_name, stop_id):
         arrival_information["trains"][station_name][stop_id][
             "estimated_times"].append(str(estimated_time) + "min")
         current_month = datetime.strftime(datetime.now(), "%b%Y")
-        file_path = main_file_path + "/cta-reliability/train_arrivals/train_arrivals-" + \
+        file_path = main_file_path + "train_arrivals/train_arrivals-" + \
             str(current_month) + ".csv"
         with open(file_path, 'a', newline='', encoding='utf8') as csvfile:
             writer_object = DictWriter(
@@ -151,7 +151,7 @@ def add_train_to_file_api(eta, station_name, stop_id):
 def check_main_train_file_exists():
     """Used to check if file exists"""
     current_month = datetime.strftime(datetime.now(), "%b%Y")
-    file_path = main_file_path + "/cta-reliability/train_arrivals/train_arrivals-" + \
+    file_path = main_file_path + "train_arrivals/train_arrivals-" + \
         str(current_month) + ".csv"
     train_csv_file = os.path.exists(file_path)
     if train_csv_file is False:
@@ -168,7 +168,7 @@ def check_main_train_file_exists():
 def check_integrity_file_exists():
     """Used to check if file exists"""
     current_month = datetime.strftime(datetime.now(), "%b%Y")
-    file_path = main_file_path + "/cta-reliability/train_arrivals/integrity-check-" + \
+    file_path = main_file_path + "train_arrivals/integrity-check-" + \
         str(current_month) + ".csv"
     integrity_csv_file = os.path.exists(file_path)
     if integrity_csv_file is False:
@@ -188,7 +188,7 @@ def add_integrity_file_line(status):
     current_simple_time = datetime.strftime(datetime.now(), "%Y-%m-%dT%H:%M")
     current_long_time = datetime.strftime(
         datetime.now(), "%Y-%m-%dT%H:%M:%S.%f%z")
-    file_path = main_file_path + "/cta-reliability/train_arrivals/integrity-check-" + \
+    file_path = main_file_path + "train_arrivals/integrity-check-" + \
         str(current_month) + ".csv"
     with open(file_path, 'a', newline='', encoding='utf8') as csvfile:
         writer_object = DictWriter(
@@ -204,7 +204,7 @@ def add_integrity_file_line(status):
     # check_backup_train_file_exists()
     check_integrity_file_exists()
     # Settings
-    file = open(file=main_file_path + '/cta-reliability/settings.json',
+    file = open(file=main_file_path + 'settings.json',
                 mode='r',
                 encoding='utf-8')
     settings = json.load(file)

apps/metra.py

@@ -19,7 +19,7 @@
 metra_password = os.getenv('METRA_PASSWORD')
 
 # Logging Information
-LOG_FILENAME = main_file_path + '/cta-reliability/logs/metra-reliability.log'
+LOG_FILENAME = main_file_path + 'logs/metra-reliability.log'
 logging.basicConfig(level=logging.INFO)
 handler = RotatingFileHandler(LOG_FILENAME, maxBytes=10e6, backupCount=10)
 formatter = logging.Formatter(
@@ -60,7 +60,7 @@ def add_train_to_file_api(response):
     current_simple_time = datetime.strftime(datetime.now(), "%Y-%m-%dT%H:%M")
     current_long_time = datetime.strftime(
         datetime.now(), "%Y-%m-%dT%H:%M:%S.%f%z")
-    file_path = main_file_path + "/cta-reliability/train_arrivals/metra_train_positions-" + \
+    file_path = main_file_path + "/train_arrivals/metra_train_positions-" + \
         str(current_month) + ".csv"
     for vehicle in response:
         with open(file_path, 'a', newline='', encoding='utf8') as csvfile:
@@ -73,7 +73,7 @@ def add_train_to_file_api(response):
 def check_main_train_file_exists():
     """Used to check if file exists"""
     current_month = datetime.strftime(datetime.now(), "%b%Y")
-    file_path = main_file_path + "/cta-reliability/train_arrivals/metra_train_positions-" + \
+    file_path = main_file_path + "/train_arrivals/metra_train_positions-" + \
         str(current_month) + ".csv"
     train_csv_file = os.path.exists(file_path)
     if train_csv_file is False:
@@ -89,7 +89,7 @@ def check_main_train_file_exists():
 def check_integrity_file_exists():
     """Used to check if file exists"""
     current_month = datetime.strftime(datetime.now(), "%b%Y")
-    file_path = main_file_path + "/cta-reliability/train_arrivals/metra-integrity-check-" + \
+    file_path = main_file_path + "/train_arrivals/metra-integrity-check-" + \
         str(current_month) + ".csv"
     integrity_csv_file = os.path.exists(file_path)
     if integrity_csv_file is False:
@@ -108,7 +108,7 @@ def add_integrity_file_line(status):
     current_simple_time = datetime.strftime(datetime.now(), "%Y-%m-%dT%H:%M")
     current_long_time = datetime.strftime(
         datetime.now(), "%Y-%m-%dT%H:%M:%S.%f%z")
-    file_path = main_file_path + "/cta-reliability/train_arrivals/metra-integrity-check-" + \
+    file_path = main_file_path + "/train_arrivals/metra-integrity-check-" + \
         str(current_month) + ".csv"
     with open(file_path, 'a', newline='', encoding='utf8') as csvfile:
         writer_object = DictWriter(
@@ -123,7 +123,7 @@ def add_integrity_file_line(status):
     check_main_train_file_exists()
     check_integrity_file_exists()
     # Settings
-    file = open(file=main_file_path + '/cta-reliability/settings.json',
+    file = open(file=main_file_path + 'settings.json',
                 mode='r',
                 encoding='utf-8')
     settings = json.load(file)

apps/station-tracking.py

@@ -125,7 +125,7 @@ def add_train_to_file_api(eta, station_name, stop_id):
         arrival_information["trains"][station_name][stop_id][
             "estimated_times"].append(str(estimated_time) + "min")
         current_month = datetime.strftime(datetime.now(), "%b%Y")
-        file_path = main_file_path + "/cta-reliability/train_arrivals/special/train_arrivals-" + \
+        file_path = main_file_path + "train_arrivals/special/train_arrivals-" + \
             str(current_month) + ".csv"
         with open(file_path, 'a', newline='', encoding='utf8') as csvfile:
             writer_object = DictWriter(
@@ -143,7 +143,7 @@ def add_train_to_file_map(destination, route, run_number, is_scheduled, predicti
     current_month = datetime.strftime(datetime.now(), "%b%Y")
     current_long_time = datetime.strftime(
         datetime.now(), "%Y-%m-%dT%H:%M:%S")
-    file_path = main_file_path + "/cta-reliability/train_arrivals/special/train_arrivals_backup-" + \
+    file_path = main_file_path + "train_arrivals/special/train_arrivals_backup-" + \
         str(current_month) + ".csv"
     with open(file_path, 'a', newline='', encoding='utf8') as csvfile:
         writer_object = DictWriter(
@@ -169,7 +169,7 @@ def train_arrival_times_map(response):
 def check_main_train_file_exists():
     """Used to check if file exists"""
     current_month = datetime.strftime(datetime.now(), "%b%Y")
-    file_path = main_file_path + "/cta-reliability/train_arrivals/special/train_arrivals-" + \
+    file_path = main_file_path + "train_arrivals/special/train_arrivals-" + \
         str(current_month) + ".csv"
     train_csv_file = os.path.exists(file_path)
     if train_csv_file is False:
@@ -185,7 +185,7 @@ def check_main_train_file_exists():
 def check_backup_train_file_exists():
     """Used to check if file exists"""
     current_month = datetime.strftime(datetime.now(), "%b%Y")
-    file_path = main_file_path + "/cta-reliability/train_arrivals/special/train_arrivals_backup-" + \
+    file_path = main_file_path + "train_arrivals/special/train_arrivals_backup-" + \
         str(current_month) + ".csv"
     train_csv_file = os.path.exists(file_path)
     if train_csv_file is False:
@@ -201,7 +201,7 @@ def check_backup_train_file_exists():
 def check_integrity_file_exists():
     """Used to check if file exists"""
     current_month = datetime.strftime(datetime.now(), "%b%Y")
-    file_path = main_file_path + "/cta-reliability/train_arrivals/special/integrity-check-" + \
+    file_path = main_file_path + "train_arrivals/special/integrity-check-" + \
         str(current_month) + ".csv"
     integrity_csv_file = os.path.exists(file_path)
     if integrity_csv_file is False:
@@ -220,7 +220,7 @@ def add_integrity_file_line(status):
     current_simple_time = datetime.strftime(datetime.now(), "%Y-%m-%dT%H:%M")
     current_long_time = datetime.strftime(
         datetime.now(), "%Y-%m-%dT%H:%M:%S.%f%z")
-    file_path = main_file_path + "/cta-reliability/train_arrivals/special/integrity-check-" + \
+    file_path = main_file_path + "train_arrivals/special/integrity-check-" + \
         str(current_month) + ".csv"
     with open(file_path, 'a', newline='', encoding='utf8') as csvfile:
         writer_object = DictWriter(
@@ -236,7 +236,7 @@ def add_integrity_file_line(status):
     check_backup_train_file_exists()
     check_integrity_file_exists()
     # Settings
-    file = open(file=main_file_path + '/cta-reliability/settings.json',
+    file = open(file=main_file_path + 'settings.json',
                 mode='r',
                 encoding='utf-8')
     settings = json.load(file)

apps/station_headways.py

@@ -26,7 +26,7 @@
 main_file_path = os.getenv('FILE_PATH')
 
 # Logging Information
-LOG_FILENAME = main_file_path + '/cta-reliability/logs/station-headways.log'
+LOG_FILENAME = main_file_path + 'logs/station-headways.log'
 logging.basicConfig(level=logging.INFO)
 handler = RotatingFileHandler(LOG_FILENAME, maxBytes=10e6, backupCount=10)
 formatter = logging.Formatter(
@@ -140,7 +140,7 @@ def output_data_to_file():
     """puts the data in a json file for the api"""
     date = datetime.strftime(datetime.utcnow(), "%Y-%m-%dT%H:%M:%SZ")
     arrival_information["last_update"] = date
-    file_path = main_file_path + "cta-reliability/train_arrivals/json/"
+    file_path = main_file_path + "train_arrivals/json/"
     with open(file_path + "special-station.json", 'w', encoding='utf-8') as file1:
         json.dump(arrival_information, file1, indent=2)
     logging.info("File Outputs Complete")
@@ -150,7 +150,7 @@ def output_data_to_file():
 # Check to make sure output file exists and write headers
 while True:  # Where the magic happens
 # Settings
-    file = open(file=main_file_path + '/cta-reliability/settings.json',
+    file = open(file=main_file_path + 'settings.json',
                 mode='r',
                 encoding='utf-8')
     settings = json.load(file)

file_export/combine_last_months_arrivals.py

@@ -10,8 +10,9 @@
 # Load .env variables
 load_dotenv()
 
-main_file_path_csv_day = os.getenv('FILE_PATH_CSV')
-main_file_path_csv_month = os.getenv('FILE_PATH_CSV_MONTH')
+main_file_path = os.getenv('FILE_PATH')
+main_file_path_csv_day = main_file_path + "train_arrivals/csv/"
+main_file_path_csv_month = main_file_path + "train_arrivals/csv_month/"
 
 
 def get_date(date_type, delay):

file_export/export_single_day_arrivals.py

@@ -13,13 +13,12 @@
 # Load .env variables
 load_dotenv()
 
-microsoft_username = os.getenv('MICROSOFT_USERNAME')
-microsoft_password = os.getenv('MICROSOFT_PASSWORD')
 microsoft_client_id = os.getenv('MICROSOFT_CLIENT_ID')
 microsoft_tenant_id = os.getenv('MICROSOFT_TENANT_ID')
 microsoft_client_secret = os.getenv('MICROSOFT_CLIENT_SECRET')
 microsoft_workspace_id = os.getenv('MICROSOFT_WORKSPACE_ID')
-main_file_path_csv = os.getenv('FILE_PATH_CSV')
+main_file_path = os.getenv('FILE_PATH')
+main_file_path_csv = main_file_path + "train_arrivals/csv/"
 cta_dataset_id = os.getenv('CTA_DATASET_ID')
 metra_dataset_id = os.getenv('METRA_DATASET_ID')
 

file_export/export_single_day_json_data.py

@@ -11,16 +11,15 @@
 # Load .env variables
 load_dotenv()
 
-microsoft_username = os.getenv('MICROSOFT_USERNAME')
-microsoft_password = os.getenv('MICROSOFT_PASSWORD')
 microsoft_client_id = os.getenv('MICROSOFT_CLIENT_ID')
 microsoft_tenant_id = os.getenv('MICROSOFT_TENANT_ID')
 microsoft_client_secret = os.getenv('MICROSOFT_CLIENT_SECRET')
 microsoft_workspace_id = os.getenv('MICROSOFT_WORKSPACE_ID')
-main_file_path_json = os.getenv('FILE_PATH_JSON')
+main_file_path = os.getenv('FILE_PATH')
 cta_dataset_id = os.getenv('CTA_DATASET_ID')
 metra_dataset_id = os.getenv('METRA_DATASET_ID')
 
+main_file_path_json = main_file_path + "train_arrivals/json/"
 
 def get_date(date_type):
     """formatted date shortcut"""

file_export/upload_files.py

@@ -18,7 +18,7 @@
 main_file_path = os.getenv('FILE_PATH')
 
 # Logging Information
-LOG_FILENAME = main_file_path + '/cta-reliability/logs/file-uploads.log'
+LOG_FILENAME = main_file_path + 'logs/file-uploads.log'
 logging.basicConfig(level=logging.INFO)
 handler = RotatingFileHandler(LOG_FILENAME, maxBytes=10e6, backupCount=10)
 formatter = logging.Formatter(
@@ -28,10 +28,10 @@
 
 
 # Constants
-MAIN_FILE_PATH_1 = "/cta-reliability/train_arrivals/train_arrivals-"
-MAIN_FILE_PATH_2 = "/cta-reliability/train_arrivals/integrity-check-"
-MAIN_FILE_PATH_3 = "/cta-reliability/train_arrivals/metra_train_positions-"
-MAIN_FILE_PATH_4 = "/cta-reliability/train_arrivals/metra-integrity-check-"
+MAIN_FILE_PATH_1 = "train_arrivals/train_arrivals-"
+MAIN_FILE_PATH_2 = "train_arrivals/integrity-check-"
+MAIN_FILE_PATH_3 = "train_arrivals/metra_train_positions-"
+MAIN_FILE_PATH_4 = "train_arrivals/metra-integrity-check-"
 
 # Dates
 current_day = datetime.strftime(datetime.now(), "%d")

settings.json

@@ -26,14 +26,10 @@
     },
     "alerts-api": {
         "api-enabled": "False",
-        "api-url": "http://lapi.transitchicago.com/api/1.0/alerts.aspx?routeid=P,Y,Blue,Pink,G,Org,Brn,BLS-1,GLS-2&outputType=JSON"
+        "api-url": "http://lapi.transitchicago.com/api/1.0/alerts.aspx?routeid=P,Y,Blue,Pink,G,Org,Brn,Red&outputType=JSON&accessibility=false"
     },
     "metra-api": {
         "api-enabled": "True",
         "api-url": "https://gtfsapi.metrarail.com/gtfs/positions"
-    },
-    "is-cta-okay":{
-        "weekday": [34, 59, 71, 86, 120, 171, 233, 312, 398, 469, 536, 602, 668, 734, 805, 878, 960, 1030, 1095, 1157, 1214, 1268, 1321, 1366],
-        "weekend": [36, 64, 78, 92, 109, 140, 182, 228, 277, 328, 381, 435, 489, 545, 601, 655, 709, 762, 815, 866, 916, 966, 1016, 1091]
     }
 }