From f123f8e8363d5d6a76bbc189ac0d40404e26c006 Mon Sep 17 00:00:00 2001 From: Arijit Roy Date: Mon, 14 Aug 2023 12:42:38 +0530 Subject: [PATCH] Seperated TV Show & TV Sries in 2 DBs --- python_scripts/mdl_invalid_record_update.py | 8 ++++---- python_scripts/mdl_psql.py | 13 +++++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/python_scripts/mdl_invalid_record_update.py b/python_scripts/mdl_invalid_record_update.py index c820dbf..55363d3 100644 --- a/python_scripts/mdl_invalid_record_update.py +++ b/python_scripts/mdl_invalid_record_update.py @@ -5,7 +5,7 @@ ETL_TIME = datetime.now().strftime("%Y-%m-%d %H:%M:%S") -DB_LIST = ['tv', 'movie'] +DB_LIST = ['tv', 'drama', 'movie'] for db_name in DB_LIST: try: @@ -14,7 +14,7 @@ fetch_query = f"select * from public.{db_name} where DATE_PART('day', localtimestamp(0) - update_ts) > 30;" df = sqlio.read_sql_query(fetch_query, conn) - print(f"{ETL_TIME} INFO: Fetched {len(df)} records for DB {db_name}") + print(f"{ETL_TIME} INFO: Fetched {len(df)} records for {db_name} DB") id_list = [] for index, row in df.iterrows(): @@ -35,9 +35,9 @@ """ cur.execute(update_query) conn.commit() - print(f"{ETL_TIME} INFO: {len(id_list)} records are updated for DB {db_name}") + print(f"{ETL_TIME} INFO: {len(id_list)} records are updated for {db_name} DB") else: - print(f"{ETL_TIME} INFO: No record to update for DB {db_name}") + print(f"{ETL_TIME} INFO: No record to update for {db_name} DB") except Exception as e: print(f"{ETL_TIME} ERROR: {str(e)}") diff --git a/python_scripts/mdl_psql.py b/python_scripts/mdl_psql.py index cb200c3..b3fe02a 100644 --- a/python_scripts/mdl_psql.py +++ b/python_scripts/mdl_psql.py @@ -92,7 +92,12 @@ def inset_into_db(content_list: list) -> int: except: rating = 99999 - table_name = "movie" if item_type == "Movie" else "tv" + if item_type == "Movie": + table_name = "movie" + elif (item_type == "Drama") or (item_type == "Special"): + table_name = "drama" + else: + table_name = "tv" fetch_query = f'SELECT * FROM {table_name} WHERE "id" = {item_id};' result = pd.read_sql(fetch_query, conn) @@ -107,7 +112,7 @@ def inset_into_db(content_list: list) -> int: INSERT INTO {table_name} ("id", "title", "episodes", "ranking", "best_ranking", "popularity", "best_popularity", - "country", "content_type", "type", "synopsis", + "country", "content_type", "record_type", "synopsis", "released_at", "url", "genres", "thumbnail", "cover", "rating", "best_rating", "insert_ts", "update_ts") VALUES @@ -143,7 +148,7 @@ def inset_into_db(content_list: list) -> int: "best_popularity" = case when ("best_popularity" > {popularity}) then {popularity} else "best_popularity" end, "country" = '{country}', "content_type" = '{content_type}', - "type" = '{item_type}', + "record_type" = '{item_type}', "synopsis" = '{synopsis}', "released_at" = '{released_at}', "url" = '{url}', @@ -170,7 +175,7 @@ def inset_into_db(content_list: list) -> int: "best_popularity" = case when ("best_popularity" > {popularity}) then {popularity} else "best_popularity" end, "country" = '{country}', "content_type" = '{content_type}', - "type" = '{item_type}', + "record_type" = '{item_type}', "released_at" = '{released_at}', "url" = '{url}', "genres" = '{genres}',