From e1dbcfb5238be5469c9a9a568c30b96cf91b74ee Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Fri, 20 Sep 2024 14:05:17 +0200 Subject: [PATCH] aws-ce-grafana-backend: add total costs per component --- .../mounted-files/const.py | 16 +++ .../mounted-files/query.py | 129 ++++++++++++++++++ .../mounted-files/webserver.py | 15 +- 3 files changed, 159 insertions(+), 1 deletion(-) diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/const.py b/helm-charts/aws-ce-grafana-backend/mounted-files/const.py index e04e69e76..8823c3593 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/const.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/const.py @@ -8,6 +8,17 @@ # values.yaml under the software configuration heading CLUSTER_NAME = os.environ["AWS_CE_GRAFANA_BACKEND__CLUSTER_NAME"] +SERVICE_COMPONENT_MAP = { + "AWS Backup": "backup", + "EC2 - Other": "compute", + "Amazon Elastic Compute Cloud - Compute": "compute", + "Amazon Elastic Container Service for Kubernetes": "fixed", + "Amazon Elastic File System": "home storage", + "Amazon Elastic Load Balancing": "networking", + "Amazon Simple Storage Service": "object storage", + "Amazon Virtual Private Cloud": "networking", +} + # Metrics: # # UnblendedCost represents costs for an individual AWS account. It is @@ -85,3 +96,8 @@ "Type": "TAG", "Key": "2i2c:hub-name", } + +GROUP_BY_SERVICE_DIMENSION = { + "Type": "DIMENSION", + "Key": "SERVICE", +} diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/query.py b/helm-charts/aws-ce-grafana-backend/mounted-files/query.py index 756ea51f9..f2007c343 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/query.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/query.py @@ -2,6 +2,9 @@ Queries to AWS Cost Explorer to get different kinds of cost data. """ +import functools +import logging + import boto3 from .const import ( @@ -9,12 +12,27 @@ FILTER_USAGE_COSTS, GRANULARITY_DAILY, GROUP_BY_HUB_TAG, + GROUP_BY_SERVICE_DIMENSION, METRICS_UNBLENDED_COST, + SERVICE_COMPONENT_MAP, ) +logger = logging.getLogger(__name__) aws_ce_client = boto3.client("ce") +@functools.cache +def _get_component_name(service_name): + print(f"test {service_name}") + logger.info(f"test {service_name}") + if service_name in SERVICE_COMPONENT_MAP: + return SERVICE_COMPONENT_MAP[service_name] + else: + # only printed once per service name thanks to memoization + logger.warning(f"Service '{service_name}' not categorized as a component yet") + return "other" + + def query_aws_cost_explorer(metrics, granularity, from_date, to_date, filter, group_by): """ Function meant to be responsible for making the API call and handling @@ -171,3 +189,114 @@ def query_total_costs_per_hub(from_date, to_date): ) return processed_response + + +def query_total_costs_per_component(from_date, to_date): + """ + A query with processing of the response tailored query to report hub + independent total costs per component - a grouping of services. + """ + response = query_aws_cost_explorer( + metrics=[METRICS_UNBLENDED_COST], + granularity=GRANULARITY_DAILY, + from_date=from_date, + to_date=to_date, + filter={ + "And": [ + FILTER_USAGE_COSTS, + FILTER_ATTRIBUTABLE_COSTS, + ] + }, + group_by=[GROUP_BY_SERVICE_DIMENSION], + ) + + # response["ResultsByTime"] is a list with entries looking like this... + # + # [ + # { + # "TimePeriod": {"Start": "2024-08-30", "End": "2024-08-31"}, + # "Total": {}, + # "Groups": [ + # { + # "Keys": ["AWS Backup"], + # "Metrics": { + # "UnblendedCost": {"Amount": "2.4763369432", "Unit": "USD"} + # }, + # }, + # { + # "Keys": ["EC2 - Other"], + # "Metrics": { + # "UnblendedCost": {"Amount": "3.2334814259", "Unit": "USD"} + # }, + # }, + # { + # "Keys": ["Amazon Elastic Compute Cloud - Compute"], + # "Metrics": { + # "UnblendedCost": {"Amount": "12.5273401469", "Unit": "USD"} + # }, + # }, + # { + # "Keys": ["Amazon Elastic Container Service for Kubernetes"], + # "Metrics": {"UnblendedCost": {"Amount": "2.4", "Unit": "USD"}}, + # }, + # { + # "Keys": ["Amazon Elastic File System"], + # "Metrics": { + # "UnblendedCost": {"Amount": "9.4433542756", "Unit": "USD"} + # }, + # }, + # { + # "Keys": ["Amazon Elastic Load Balancing"], + # "Metrics": { + # "UnblendedCost": {"Amount": "0.6147035689", "Unit": "USD"} + # }, + # }, + # { + # "Keys": ["Amazon Simple Storage Service"], + # "Metrics": { + # "UnblendedCost": {"Amount": "0.1094078516", "Unit": "USD"} + # }, + # }, + # { + # "Keys": ["Amazon Virtual Private Cloud"], + # "Metrics": { + # "UnblendedCost": {"Amount": "0.24867778", "Unit": "USD"} + # }, + # }, + # ], + # "Estimated": False, + # }, + # ] + # + # processed_response is a list with entries looking like this... + # + # [ + # { + # "date": "2024-08-30", + # "cost": "12.19", + # "name": "home storage", + # }, + # ] + # + processed_response = [] + for e in response["ResultsByTime"]: + # coalesce service costs to component costs + component_costs = {} + for g in e["Groups"]: + service_name = g["Keys"][0] + name = _get_component_name(service_name) + cost = float(g["Metrics"]["UnblendedCost"]["Amount"]) + component_costs[name] = component_costs.get(name, 0.0) + cost + + processed_response.extend( + [ + { + "date": e["TimePeriod"]["Start"], + "cost": f"{cost:.2f}", + "name": name, + } + for name, cost in component_costs.items() + ] + ) + + return processed_response diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py b/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py index d8ef70345..e2725934a 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py @@ -1,10 +1,16 @@ +import logging from datetime import date, datetime, timedelta, timezone from flask import Flask, request -from .query import query_total_costs, query_total_costs_per_hub +from .query import ( + query_total_costs, + query_total_costs_per_component, + query_total_costs_per_hub, +) app = Flask(__name__) +logging.basicConfig(level=logging.INFO) def parse_from_to_in_query_params(): @@ -54,3 +60,10 @@ def total_costs_per_hub(): from_date, to_date = parse_from_to_in_query_params() return query_total_costs_per_hub(from_date, to_date) + + +@app.route("/total-costs-per-component") +def total_costs_per_component(): + from_date, to_date = parse_from_to_in_query_params() + + return query_total_costs_per_component(from_date, to_date)