aws-ce-grafana-backend: add per hub repeating component panel, and cache better

helm-charts/aws-ce-grafana-backend/mounted-files/cache.py

@@ -0,0 +1,32 @@
+"""
+It seems Python doesn't provide a built in way to cache function calls, so this
+implements an answer from stackoverflow by user McToel from
+https://stackoverflow.com/a/73026174/2220152.
+
+A limitation of this is implementation is that it can't be used on functions
+accepting lists etc, even if the functions we have only have lists with hashable
+content that could be compared and concluded equal.
+"""
+
+import time
+from functools import lru_cache
+
+
+def ttl_lru_cache(seconds_to_live: int, maxsize: int = 128):
+    """
+    Time aware lru caching
+    """
+
+    def wrapper(func):
+
+        @lru_cache(maxsize)
+        def inner(__ttl, *args, **kwargs):
+            # Note that __ttl is not passed down to func,
+            # as it's only used to trigger cache miss after some time
+            return func(*args, **kwargs)
+
+        return lambda *args, **kwargs: inner(
+            time.time() // seconds_to_live, *args, **kwargs
+        )
+
+    return wrapper

helm-charts/aws-ce-grafana-backend/mounted-files/query.py

@@ -7,6 +7,7 @@
 
 import boto3
 
+from .cache import ttl_lru_cache
 from .const import (
     FILTER_ATTRIBUTABLE_COSTS,
     FILTER_USAGE_COSTS,
@@ -23,8 +24,6 @@
 
 @functools.cache
 def _get_component_name(service_name):
-    print(f"test {service_name}")
-    logger.info(f"test {service_name}")
     if service_name in SERVICE_COMPONENT_MAP:
         return SERVICE_COMPONENT_MAP[service_name]
     else:
@@ -49,6 +48,39 @@ def query_aws_cost_explorer(metrics, granularity, from_date, to_date, filter, gr
     return response
 
 
+@ttl_lru_cache(seconds_to_live=3600)
+def query_hub_names(from_date, to_date):
+    # ref: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ce/client/get_tags.html
+    response = aws_ce_client.get_tags(
+        TimePeriod={"Start": from_date, "End": to_date},
+        TagKey="2i2c:hub-name",
+    )
+    # response looks like...
+    #
+    # {
+    #     "Tags": ["", "prod", "staging", "workshop"],
+    #     "ReturnSize": 4,
+    #     "TotalSize": 4,
+    #     "ResponseMetadata": {
+    #         "RequestId": "23736d32-9929-4b6a-8c4f-d80b1487ed37",
+    #         "HTTPStatusCode": 200,
+    #         "HTTPHeaders": {
+    #             "date": "Fri, 20 Sep 2024 12:42:13 GMT",
+    #             "content-type": "application/x-amz-json-1.1",
+    #             "content-length": "70",
+    #             "connection": "keep-alive",
+    #             "x-amzn-requestid": "23736d32-9929-4b6a-8c4f-d80b1487ed37",
+    #             "cache-control": "no-cache",
+    #         },
+    #         "RetryAttempts": 0,
+    #     },
+    # }
+    #
+    hub_names = [t or "shared" for t in response["Tags"]]
+    return hub_names
+
+
+@ttl_lru_cache(seconds_to_live=3600)
 def query_total_costs(from_date, to_date):
     """
     A query with processing of the response tailored query to report hub
@@ -107,6 +139,7 @@ def query_total_costs(from_date, to_date):
     return processed_response
 
 
+@ttl_lru_cache(seconds_to_live=3600)
 def query_total_costs_per_hub(from_date, to_date):
     """
     A query with processing of the response tailored query to report total costs
@@ -191,22 +224,47 @@ def query_total_costs_per_hub(from_date, to_date):
     return processed_response
 
 
-def query_total_costs_per_component(from_date, to_date):
+@ttl_lru_cache(seconds_to_live=3600)
+def query_total_costs_per_component(from_date, to_date, hub_name=None):
     """
-    A query with processing of the response tailored query to report hub
-    independent total costs per component - a grouping of services.
+    A query with processing of the response tailored query to report total costs
+    per component - a grouping of services.
+
+    If a hub_name is specified, component costs are filtered to only consider
+    costs directly attributable to the hub name.
     """
+    filter = {
+        "And": [
+            FILTER_USAGE_COSTS,
+            FILTER_ATTRIBUTABLE_COSTS,
+        ]
+    }
+    if hub_name == "shared":
+        filter["And"].append(
+            {
+                "Tags": {
+                    "Key": "2i2c:hub-name",
+                    "MatchOptions": ["ABSENT"],
+                },
+            }
+        )
+    elif hub_name:
+        filter["And"].append(
+            {
+                "Tags": {
+                    "Key": "2i2c:hub-name",
+                    "Values": [hub_name],
+                    "MatchOptions": ["EQUALS"],
+                },
+            }
+        )
+
     response = query_aws_cost_explorer(
         metrics=[METRICS_UNBLENDED_COST],
         granularity=GRANULARITY_DAILY,
         from_date=from_date,
         to_date=to_date,
-        filter={
-            "And": [
-                FILTER_USAGE_COSTS,
-                FILTER_ATTRIBUTABLE_COSTS,
-            ]
-        },
+        filter=filter,
         group_by=[GROUP_BY_SERVICE_DIMENSION],
     )
 

helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py

@@ -4,6 +4,7 @@
 from flask import Flask, request
 
 from .query import (
+    query_hub_names,
     query_total_costs,
     query_total_costs_per_component,
     query_total_costs_per_hub,
@@ -13,7 +14,7 @@
 logging.basicConfig(level=logging.INFO)
 
 
-def parse_from_to_in_query_params():
+def _parse_from_to_in_query_params():
     """
     Parse "from" and "to" query parameters, expected to arrive as YYYY-MM-DD
     strings.
@@ -48,22 +49,30 @@ def ready():
     return ("", 204)
 
 
+@app.route("/hub-names")
+def hub_names():
+    from_date, to_date = _parse_from_to_in_query_params()
+
+    return query_hub_names(from_date, to_date)
+
+
 @app.route("/total-costs")
 def total_costs():
-    from_date, to_date = parse_from_to_in_query_params()
+    from_date, to_date = _parse_from_to_in_query_params()
 
     return query_total_costs(from_date, to_date)
 
 
 @app.route("/total-costs-per-hub")
 def total_costs_per_hub():
-    from_date, to_date = parse_from_to_in_query_params()
+    from_date, to_date = _parse_from_to_in_query_params()
 
     return query_total_costs_per_hub(from_date, to_date)
 
 
 @app.route("/total-costs-per-component")
 def total_costs_per_component():
-    from_date, to_date = parse_from_to_in_query_params()
+    from_date, to_date = _parse_from_to_in_query_params()
+    hub_name = request.args.get("hub")
 
-    return query_total_costs_per_component(from_date, to_date)
+    return query_total_costs_per_component(from_date, to_date, hub_name)