Skip to content

Commit

Permalink
Added logs on the controller of the data quality tool.
Browse files Browse the repository at this point in the history
  • Loading branch information
KFilippopolitis committed Jun 20, 2024
1 parent e8ab00f commit b74babb
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 13 deletions.
2 changes: 1 addition & 1 deletion data_quality_tool/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,5 @@ services:
context: ./frontend
dockerfile: Dockerfile
ports:
- "8080:80"
- "80:80"

41 changes: 29 additions & 12 deletions data_quality_tool/controller.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
import json

from flask import Flask, request, jsonify, send_file
from flask_cors import CORS
import pandas as pd
from io import BytesIO

from converter.excel_to_json import convert_excel_to_json
from converter.json_to_excel import convert_json_to_excel
from validator import json_validator, excel_validator
Expand All @@ -14,27 +12,31 @@

@app.route("/")
def home():
print("Home endpoint accessed")
return "Welcome to the Excel-JSON Converter API!"


@app.route("/excel-to-json", methods=["POST"])
def excel_to_json():
print("excel_to_json endpoint accessed")
if "file" not in request.files:
print("No file part in request")
return jsonify({"error": "No file part"}), 400
file = request.files["file"]
if file.filename == "":
print("No selected file")
return jsonify({"error": "No selected file"}), 400
if file:
print(f"Processing file: {file.filename}")
file_stream = BytesIO(file.read())
# Use the file_stream object with pandas
df = pd.read_excel(file_stream, engine="openpyxl")
print("Excel file read into DataFrame")
excel_validator.validate_excel(df)
# Read the Excel file into a Pandas DataFrame
print("Excel file validated")
json_data = convert_excel_to_json(df)

# Pretty-print the JSON data
print("Excel file converted to JSON")
pretty_json_data = json.dumps(json_data, indent=4)

print("JSON data pretty-printed")
response = app.response_class(
response=pretty_json_data,
status=200,
Expand All @@ -44,51 +46,66 @@ def excel_to_json():

@app.route("/json-to-excel", methods=["POST"])
def json_to_excel():
print("json_to_excel endpoint accessed")
if not request.json:
print("No JSON provided in request")
return "Please provide the json", 400
json_data = request.json
df = convert_json_to_excel(json_data)
print(f"Processing JSON data: {json_data}")
json_validator.validate_json(json_data)
# Create a BytesIO buffer to save the Excel file
print("JSON data validated")
df = convert_json_to_excel(json_data)
print("JSON data converted to Excel")
output = BytesIO()
with pd.ExcelWriter(output, engine="xlsxwriter") as writer:
df.to_excel(writer, index=False)
output.seek(0)
# Send the generated Excel file
print("Excel file created and saved to buffer")
return send_file(
output,
as_attachment=True,
download_name="output.xlsx", # Use download_name for newer Flask versions if attachment_filename causes issues
download_name="output.xlsx",
mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
)

@app.route("/validate-json", methods=["POST"])
def validate_json():
print("validate_json endpoint accessed")
json_data = request.json
if not request.json:
print("No JSON provided in request")
return "Please provide the json", 400
try:
json_validator.validate_json(json_data)
print("JSON data is valid")
return jsonify({"message": "Data model is valid."})
except json_validator.InvalidDataModelError as e:
print(f"JSON validation error: {str(e)}")
return jsonify({"error": str(e)}), 400

@app.route("/validate-excel", methods=["POST"])
def validate_excel():
print("validate_excel endpoint accessed")
if "file" not in request.files:
print("No file part in request")
return jsonify({"error": "No file part"}), 400
file = request.files["file"]
if file.filename == "":
print("No selected file")
return jsonify({"error": "No selected file"}), 400
if file:
print(f"Processing file: {file.filename}")
file_stream = BytesIO(file.read())
# Use the file_stream object with pandas
df = pd.read_excel(file_stream, engine="openpyxl")
print("Excel file read into DataFrame")
try:
excel_validator.validate_excel(df)
print("Excel file is valid")
return jsonify({"message": "Data model is valid."})
except json_validator.InvalidDataModelError as e:
print(f"Excel validation error: {str(e)}")
return jsonify({"error": str(e)}), 400

if __name__ == '__main__':
print("Starting Flask server...")
app.run(host='0.0.0.0', port=8000)

0 comments on commit b74babb

Please sign in to comment.