store metrics user password in relation data

- don't log full introspection command - don't want to log passwords

metadata.yaml

@@ -2,7 +2,7 @@
 # Licensed under the GPLv3, see LICENSE file for details.
 name: juju-controller
 assumes:
-- juju >= 3.0
+- juju >= 3.0 # TODO: update to 3.2/3.3
 description: |
   The Juju controller charm is used to expose various pieces
   of functionality of a Juju controller.

src/charm.py

@@ -9,9 +9,11 @@
 import yaml
 from ops.charm import CharmBase
 from ops.framework import StoredState
+from ops.charm import RelationJoinedEvent, RelationDepartedEvent
 from ops.main import main
-from ops.model import ActiveStatus, BlockedStatus
+from ops.model import ActiveStatus, BlockedStatus, Relation, Unit
 from charms.prometheus_k8s.v0.prometheus_scrape import MetricsEndpointProvider
+from typing import MutableMapping
 
 logger = logging.getLogger(__name__)
 
@@ -28,16 +30,10 @@ def __init__(self, *args):
         self.framework.observe(
             self.on.website_relation_joined, self._on_website_relation_joined)
 
-        # Set up Prometheus integration
-        # Store user credentials used to access Juju's metrics endpoint.
-        self._stored.set_default(metrics_users=dict())
-        self.metrics_endpoint = MetricsEndpointProvider(self,
-            jobs = self._prometheus_jobs()
-        )
         self.framework.observe(
-            self.on.metrics_endpoint_relation_joined, self._on_metrics_endpoint_relation_joined)
+            self.on.metrics_endpoint_relation_created, self._on_metrics_endpoint_relation_created)
         self.framework.observe(
-            self.on.metrics_endpoint_relation_departed, self._on_metrics_endpoint_relation_departed)
+            self.on.metrics_endpoint_relation_broken, self._on_metrics_endpoint_relation_broken)
 
     def _on_start(self, _):
         self.unit.status = ActiveStatus()
@@ -77,20 +73,38 @@ def _on_website_relation_joined(self, event):
                     'port': str(port)
                 })
 
-    def _on_metrics_endpoint_relation_joined(self, event):
-        # Add new user to access metrics
-        username = metrics_username(event.relation)
-        password = secrets.token_urlsafe(16)
-        add_metrics_user(username, password)
-        self._stored.metrics_users[username] = password
-        self.metrics_endpoint.update_scrape_job_spec(self._prometheus_jobs())
+    def _on_metrics_endpoint_relation_created(self, event: RelationJoinedEvent):
+        # Ensure that user credentials exist to access the metrics endpoint.
+        # We've done it this way because it's possible for this hook to run twice.
+        logger.info(f'relation data: {event.relation.data[self.unit]}')
+        username = metrics_username(event.relation, self.unit)
+        password = ensure_metrics_user(username, event.relation.data[self.unit])
+
+        # Set up Prometheus scrape config
+        self.metrics_endpoint = MetricsEndpointProvider(self,
+            jobs = [{
+                # "job_name": "juju",
+                "metrics_path": "/introspection/metrics",
+                "scheme": "https",
+                "static_configs": [{"targets": ["*:17070"]}],
+                "basic_auth": {
+                    "username": f'user-{username}',
+                    "password": password,
+                },
+                "tls_config": {
+                    "ca_file": ca_cert(),
+                    "server_name": "juju-apiserver",
+                },
+            }],
+        )
+        self.metrics_endpoint.set_scrape_job_spec()
 
-    def _on_metrics_endpoint_relation_departed(self, event):
+    def _on_metrics_endpoint_relation_broken(self, event: RelationDepartedEvent):
         # Remove metrics user
-        username = metrics_username(event.relation)
+        username = metrics_username(event.relation, self.unit)
         remove_metrics_user(username)
-        del self._stored.metrics_users[username]
-        self.metrics_endpoint.update_scrape_job_spec(self._prometheus_jobs())
+
+        # self.metrics_endpoint.update_scrape_job_spec(self._prometheus_jobs())
 
     def _prometheus_jobs(self):
         '''
@@ -140,38 +154,66 @@ def ca_cert() -> str:
     '''
     return _agent_conf('cacert')
 
-def metrics_username(relation) -> str:
+def metrics_username(relation: Relation, unit: Unit) -> str:
     '''
     metrics_username returns the username used to access the metrics endpoint,
-    for the given relation.
+    for the given relation and unit. This username has the form
+        juju-metrics-u0-r1
     '''
-    return f'juju-metrics-{relation.id}'
+    unit_number = unit.name.split('/')[1]
+    return f'juju-metrics-u{unit_number}-r{relation.id}'
 
 def _introspect(command: str):
     '''
     Runs an introspection command inside the controller machine.
     '''
-    logger.info(f'running introspect command {command}')
-    result = subprocess.run(
-        f"source /etc/profile.d/juju-introspection.sh && {command}",
-        shell=True,
-        executable="/bin/bash",
-        stdout=subprocess.PIPE,
-    )
+    try:
+        result = subprocess.run(
+            f"source /etc/profile.d/juju-introspection.sh && {command}",
+            shell=True,
+            executable="/bin/bash",
+            stdout=subprocess.PIPE,
+        )
+    except BaseException as e:
+        logger.error(f"introspect command failed: {e}", exc_info=1)
     logger.info(f'stdout: {result.stdout}')
 
-def add_metrics_user(username: str, password: str):
+def _add_metrics_user(username: str, password: str):
     '''
     Runs the following introspection command:
         juju_add_metrics_user <username> <password>
     '''
+    logger.info(f'adding metrics user {username}')
     _introspect(f"juju_add_metrics_user {username} {password}")
 
+def ensure_metrics_user(username: str, relation_data: MutableMapping[str, str]) -> str:
+    '''
+    Ensures a metrics user with the given username exists.
+    If the user exists, return their password as stored in relation data.
+    If not, create the new user via the introspection endpoint, store their
+    password in relation data, and return the new password.
+    This function is idempotent.
+    ''' 
+    metrics_password_key = "metrics_password"
+
+    # Check if user exists in relation data
+    if metrics_password_key in relation_data:
+        logger.debug(f'metrics user password found in relation data')
+        return relation_data[metrics_password_key]
+
+    # Create new user
+    logger.debug(f'no password found in relation data, creating new metrics user')
+    password = secrets.token_urlsafe(16)
+    _add_metrics_user(username, password)
+    relation_data[metrics_password_key] = password
+    return password
+
 def remove_metrics_user(username: str):
     '''
     Runs the following introspection command:
         juju_remove_metrics_user <username>
     '''
+    logger.info(f'removing metrics user {username}')
     _introspect(f"juju_remove_metrics_user {username}")
 
 if __name__ == "__main__":