From 468c692dff1acce7067b20ad0f4c2719ba786a6e Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Fri, 20 Sep 2024 14:15:39 +0200 Subject: [PATCH 1/2] aws-ce-grafana-backend: cleanup mistakenly introduced debugging statements --- helm-charts/aws-ce-grafana-backend/mounted-files/query.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/query.py b/helm-charts/aws-ce-grafana-backend/mounted-files/query.py index f2007c343..13153fd55 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/query.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/query.py @@ -23,8 +23,6 @@ @functools.cache def _get_component_name(service_name): - print(f"test {service_name}") - logger.info(f"test {service_name}") if service_name in SERVICE_COMPONENT_MAP: return SERVICE_COMPONENT_MAP[service_name] else: From 75a14bbbdc8937f25e8867771b071b497ae98515 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Fri, 20 Sep 2024 15:29:13 +0200 Subject: [PATCH 2/2] aws-ce-grafana-backend: add per hub repeating component panel, and cache better --- .../mounted-files/cache.py | 32 ++++++++ .../mounted-files/query.py | 78 ++++++++++++++++--- .../mounted-files/webserver.py | 19 +++-- 3 files changed, 115 insertions(+), 14 deletions(-) create mode 100644 helm-charts/aws-ce-grafana-backend/mounted-files/cache.py diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/cache.py b/helm-charts/aws-ce-grafana-backend/mounted-files/cache.py new file mode 100644 index 000000000..8a39f0ce4 --- /dev/null +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/cache.py @@ -0,0 +1,32 @@ +""" +It seems Python doesn't provide a built in way to cache function calls, so this +implements an answer from stackoverflow by user McToel from +https://stackoverflow.com/a/73026174/2220152. + +A limitation of this is implementation is that it can't be used on functions +accepting lists etc, even if the functions we have only have lists with hashable +content that could be compared and concluded equal. +""" + +import time +from functools import lru_cache + + +def ttl_lru_cache(seconds_to_live: int, maxsize: int = 128): + """ + Time aware lru caching + """ + + def wrapper(func): + + @lru_cache(maxsize) + def inner(__ttl, *args, **kwargs): + # Note that __ttl is not passed down to func, + # as it's only used to trigger cache miss after some time + return func(*args, **kwargs) + + return lambda *args, **kwargs: inner( + time.time() // seconds_to_live, *args, **kwargs + ) + + return wrapper diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/query.py b/helm-charts/aws-ce-grafana-backend/mounted-files/query.py index 13153fd55..9c7977549 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/query.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/query.py @@ -7,6 +7,7 @@ import boto3 +from .cache import ttl_lru_cache from .const import ( FILTER_ATTRIBUTABLE_COSTS, FILTER_USAGE_COSTS, @@ -47,6 +48,39 @@ def query_aws_cost_explorer(metrics, granularity, from_date, to_date, filter, gr return response +@ttl_lru_cache(seconds_to_live=3600) +def query_hub_names(from_date, to_date): + # ref: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ce/client/get_tags.html + response = aws_ce_client.get_tags( + TimePeriod={"Start": from_date, "End": to_date}, + TagKey="2i2c:hub-name", + ) + # response looks like... + # + # { + # "Tags": ["", "prod", "staging", "workshop"], + # "ReturnSize": 4, + # "TotalSize": 4, + # "ResponseMetadata": { + # "RequestId": "23736d32-9929-4b6a-8c4f-d80b1487ed37", + # "HTTPStatusCode": 200, + # "HTTPHeaders": { + # "date": "Fri, 20 Sep 2024 12:42:13 GMT", + # "content-type": "application/x-amz-json-1.1", + # "content-length": "70", + # "connection": "keep-alive", + # "x-amzn-requestid": "23736d32-9929-4b6a-8c4f-d80b1487ed37", + # "cache-control": "no-cache", + # }, + # "RetryAttempts": 0, + # }, + # } + # + hub_names = [t or "shared" for t in response["Tags"]] + return hub_names + + +@ttl_lru_cache(seconds_to_live=3600) def query_total_costs(from_date, to_date): """ A query with processing of the response tailored query to report hub @@ -105,6 +139,7 @@ def query_total_costs(from_date, to_date): return processed_response +@ttl_lru_cache(seconds_to_live=3600) def query_total_costs_per_hub(from_date, to_date): """ A query with processing of the response tailored query to report total costs @@ -189,22 +224,47 @@ def query_total_costs_per_hub(from_date, to_date): return processed_response -def query_total_costs_per_component(from_date, to_date): +@ttl_lru_cache(seconds_to_live=3600) +def query_total_costs_per_component(from_date, to_date, hub_name=None): """ - A query with processing of the response tailored query to report hub - independent total costs per component - a grouping of services. + A query with processing of the response tailored query to report total costs + per component - a grouping of services. + + If a hub_name is specified, component costs are filtered to only consider + costs directly attributable to the hub name. """ + filter = { + "And": [ + FILTER_USAGE_COSTS, + FILTER_ATTRIBUTABLE_COSTS, + ] + } + if hub_name == "shared": + filter["And"].append( + { + "Tags": { + "Key": "2i2c:hub-name", + "MatchOptions": ["ABSENT"], + }, + } + ) + elif hub_name: + filter["And"].append( + { + "Tags": { + "Key": "2i2c:hub-name", + "Values": [hub_name], + "MatchOptions": ["EQUALS"], + }, + } + ) + response = query_aws_cost_explorer( metrics=[METRICS_UNBLENDED_COST], granularity=GRANULARITY_DAILY, from_date=from_date, to_date=to_date, - filter={ - "And": [ - FILTER_USAGE_COSTS, - FILTER_ATTRIBUTABLE_COSTS, - ] - }, + filter=filter, group_by=[GROUP_BY_SERVICE_DIMENSION], ) diff --git a/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py b/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py index e2725934a..f0baaab6f 100644 --- a/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py +++ b/helm-charts/aws-ce-grafana-backend/mounted-files/webserver.py @@ -4,6 +4,7 @@ from flask import Flask, request from .query import ( + query_hub_names, query_total_costs, query_total_costs_per_component, query_total_costs_per_hub, @@ -13,7 +14,7 @@ logging.basicConfig(level=logging.INFO) -def parse_from_to_in_query_params(): +def _parse_from_to_in_query_params(): """ Parse "from" and "to" query parameters, expected to arrive as YYYY-MM-DD strings. @@ -48,22 +49,30 @@ def ready(): return ("", 204) +@app.route("/hub-names") +def hub_names(): + from_date, to_date = _parse_from_to_in_query_params() + + return query_hub_names(from_date, to_date) + + @app.route("/total-costs") def total_costs(): - from_date, to_date = parse_from_to_in_query_params() + from_date, to_date = _parse_from_to_in_query_params() return query_total_costs(from_date, to_date) @app.route("/total-costs-per-hub") def total_costs_per_hub(): - from_date, to_date = parse_from_to_in_query_params() + from_date, to_date = _parse_from_to_in_query_params() return query_total_costs_per_hub(from_date, to_date) @app.route("/total-costs-per-component") def total_costs_per_component(): - from_date, to_date = parse_from_to_in_query_params() + from_date, to_date = _parse_from_to_in_query_params() + hub_name = request.args.get("hub") - return query_total_costs_per_component(from_date, to_date) + return query_total_costs_per_component(from_date, to_date, hub_name)