Merge pull request #4847 from consideRatio/pr/aws-ce-per-comp

aws-ce-grafana-backend: add total costs per component

helm-charts/aws-ce-grafana-backend/mounted-files/const.py

@@ -8,6 +8,17 @@
 # values.yaml under the software configuration heading
 CLUSTER_NAME = os.environ["AWS_CE_GRAFANA_BACKEND__CLUSTER_NAME"]
 
+SERVICE_COMPONENT_MAP = {
+    "AWS Backup": "backup",
+    "EC2 - Other": "compute",
+    "Amazon Elastic Compute Cloud - Compute": "compute",
+    "Amazon Elastic Container Service for Kubernetes": "fixed",
+    "Amazon Elastic File System": "home storage",
+    "Amazon Elastic Load Balancing": "networking",
+    "Amazon Simple Storage Service": "object storage",
+    "Amazon Virtual Private Cloud": "networking",
+}
+
 # Metrics:
 #
 #   UnblendedCost represents costs for an individual AWS account. It is
@@ -85,3 +96,8 @@
     "Type": "TAG",
     "Key": "2i2c:hub-name",
 }
+
+GROUP_BY_SERVICE_DIMENSION = {
+    "Type": "DIMENSION",
+    "Key": "SERVICE",
+}

helm-charts/aws-ce-grafana-backend/mounted-files/query.py

@@ -2,19 +2,37 @@
 Queries to AWS Cost Explorer to get different kinds of cost data.
 """
 
+import functools
+import logging
+
 import boto3
 
 from .const import (
     FILTER_ATTRIBUTABLE_COSTS,
     FILTER_USAGE_COSTS,
     GRANULARITY_DAILY,
     GROUP_BY_HUB_TAG,
+    GROUP_BY_SERVICE_DIMENSION,
     METRICS_UNBLENDED_COST,
+    SERVICE_COMPONENT_MAP,
 )
 
+logger = logging.getLogger(__name__)
 aws_ce_client = boto3.client("ce")
 
 
+@functools.cache
+def _get_component_name(service_name):
+    print(f"test {service_name}")
+    logger.info(f"test {service_name}")
+    if service_name in SERVICE_COMPONENT_MAP:
+        return SERVICE_COMPONENT_MAP[service_name]
+    else:
+        # only printed once per service name thanks to memoization
+        logger.warning(f"Service '{service_name}' not categorized as a component yet")
+        return "other"
+
+
 def query_aws_cost_explorer(metrics, granularity, from_date, to_date, filter, group_by):
     """
     Function meant to be responsible for making the API call and handling
@@ -171,3 +189,114 @@ def query_total_costs_per_hub(from_date, to_date):
         )
 
     return processed_response
+
+
+def query_total_costs_per_component(from_date, to_date):
+    """
+    A query with processing of the response tailored query to report hub
+    independent total costs per component - a grouping of services.
+    """
+    response = query_aws_cost_explorer(
+        metrics=[METRICS_UNBLENDED_COST],
+        granularity=GRANULARITY_DAILY,
+        from_date=from_date,
+        to_date=to_date,
+        filter={
+            "And": [
+                FILTER_USAGE_COSTS,
+                FILTER_ATTRIBUTABLE_COSTS,
+            ]
+        },
+        group_by=[GROUP_BY_SERVICE_DIMENSION],
+    )
+
+    # response["ResultsByTime"] is a list with entries looking like this...
+    #
+    # [
+    #     {
+    #         "TimePeriod": {"Start": "2024-08-30", "End": "2024-08-31"},
+    #         "Total": {},
+    #         "Groups": [
+    #             {
+    #                 "Keys": ["AWS Backup"],
+    #                 "Metrics": {
+    #                     "UnblendedCost": {"Amount": "2.4763369432", "Unit": "USD"}
+    #                 },
+    #             },
+    #             {
+    #                 "Keys": ["EC2 - Other"],
+    #                 "Metrics": {
+    #                     "UnblendedCost": {"Amount": "3.2334814259", "Unit": "USD"}
+    #                 },
+    #             },
+    #             {
+    #                 "Keys": ["Amazon Elastic Compute Cloud - Compute"],
+    #                 "Metrics": {
+    #                     "UnblendedCost": {"Amount": "12.5273401469", "Unit": "USD"}
+    #                 },
+    #             },
+    #             {
+    #                 "Keys": ["Amazon Elastic Container Service for Kubernetes"],
+    #                 "Metrics": {"UnblendedCost": {"Amount": "2.4", "Unit": "USD"}},
+    #             },
+    #             {
+    #                 "Keys": ["Amazon Elastic File System"],
+    #                 "Metrics": {
+    #                     "UnblendedCost": {"Amount": "9.4433542756", "Unit": "USD"}
+    #                 },
+    #             },
+    #             {
+    #                 "Keys": ["Amazon Elastic Load Balancing"],
+    #                 "Metrics": {
+    #                     "UnblendedCost": {"Amount": "0.6147035689", "Unit": "USD"}
+    #                 },
+    #             },
+    #             {
+    #                 "Keys": ["Amazon Simple Storage Service"],
+    #                 "Metrics": {
+    #                     "UnblendedCost": {"Amount": "0.1094078516", "Unit": "USD"}
+    #                 },
+    #             },
+    #             {
+    #                 "Keys": ["Amazon Virtual Private Cloud"],
+    #                 "Metrics": {
+    #                     "UnblendedCost": {"Amount": "0.24867778", "Unit": "USD"}
+    #                 },
+    #             },
+    #         ],
+    #         "Estimated": False,
+    #     },
+    # ]
+    #
+    # processed_response is a list with entries looking like this...
+    #
+    # [
+    #     {
+    #         "date": "2024-08-30",
+    #         "cost": "12.19",
+    #         "name": "home storage",
+    #     },
+    # ]
+    #
+    processed_response = []
+    for e in response["ResultsByTime"]:
+        # coalesce service costs to component costs
+        component_costs = {}
+        for g in e["Groups"]:
+            service_name = g["Keys"][0]
+            name = _get_component_name(service_name)
+            cost = float(g["Metrics"]["UnblendedCost"]["Amount"])
+            component_costs[name] = component_costs.get(name, 0.0) + cost
+
+        processed_response.extend(
+            [
+                {
+                    "date": e["TimePeriod"]["Start"],
+                    "cost": f"{cost:.2f}",
+                    "name": name,
+                }
+                for name, cost in component_costs.items()
+            ]
+        )
+
+    return processed_response

helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py

@@ -1,10 +1,16 @@
+import logging
 from datetime import date, datetime, timedelta, timezone
 
 from flask import Flask, request
 
-from .query import query_total_costs, query_total_costs_per_hub
+from .query import (
+    query_total_costs,
+    query_total_costs_per_component,
+    query_total_costs_per_hub,
+)
 
 app = Flask(__name__)
+logging.basicConfig(level=logging.INFO)
 
 
 def parse_from_to_in_query_params():
@@ -54,3 +60,10 @@ def total_costs_per_hub():
     from_date, to_date = parse_from_to_in_query_params()
 
     return query_total_costs_per_hub(from_date, to_date)
+
+
+@app.route("/total-costs-per-component")
+def total_costs_per_component():
+    from_date, to_date = parse_from_to_in_query_params()
+
+    return query_total_costs_per_component(from_date, to_date)