Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Health monitor #112

Merged
merged 42 commits into from
Apr 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
d369fbb
Make the logging nicer-looking
breuleux Mar 7, 2024
02f600d
Basic data structures for alerts
breuleux Mar 8, 2024
ff4227c
First version of the runner
breuleux Mar 8, 2024
4ceb48c
Add gifnoc and watchdog dependencies
breuleux Mar 8, 2024
a7d53cb
Add depends field to checks
breuleux Mar 8, 2024
279e038
First version of the monitor
breuleux Mar 8, 2024
d6656c3
Add `sarc health` command
breuleux Mar 8, 2024
30d0441
Add check parameterization
breuleux Mar 11, 2024
5d86956
Add jobs_last_hour and jobs_last_week as fixtures
breuleux Mar 11, 2024
d39c48e
Add FilterCheck
breuleux Mar 11, 2024
80403f7
Implement spancache
breuleux Mar 12, 2024
3e421d3
Rewrite FilterCheck
breuleux Mar 12, 2024
c78445a
Improve ran_for
breuleux Mar 13, 2024
a137dfc
Checks can output multiple statuses
breuleux Mar 13, 2024
6565e33
Add latest_job_series
breuleux Mar 13, 2024
0db1c0e
Move fixtures.py to fetch.py
breuleux Mar 13, 2024
d4971d9
Add jobs argument to load_job_series
breuleux Mar 14, 2024
14588c5
Add bound method to cache
breuleux Mar 25, 2024
a20c8f7
Add health history command
breuleux Mar 25, 2024
f96ab22
fixup! Add bound method to cache
breuleux Mar 25, 2024
8ea5743
Misc fixes in alerts.common
breuleux Mar 25, 2024
a12f393
Store check information with the check result
breuleux Mar 25, 2024
05736d7
Test alerts.cache
breuleux Mar 25, 2024
9770fcb
Test alerts.common and alerts.runner
breuleux Mar 28, 2024
3bae088
Test monitor
breuleux Mar 28, 2024
0733e47
Remove checks, will add in other PR
breuleux Apr 2, 2024
ceb7905
Unconditional return in HealthCheckCommand
breuleux Apr 2, 2024
e99f59b
Revert series
breuleux Apr 2, 2024
9631131
Remove fetch (put back in next PR)
breuleux Apr 2, 2024
af09b89
Update gifnoc
breuleux Apr 2, 2024
84b1a6e
Add documentation
breuleux Apr 2, 2024
af21bb5
Document runner and monitor
breuleux Apr 2, 2024
60a688c
Shut linter up
breuleux Apr 2, 2024
4fb694e
Rewrite annotation
breuleux Apr 2, 2024
23b88ee
Fix datetime serialization
breuleux Apr 2, 2024
aa43d91
Use Union instead of |
breuleux Apr 2, 2024
9d62866
Force downgrade of pandas
breuleux Apr 2, 2024
e9e2416
Remove match statements
breuleux Apr 2, 2024
f0193fd
Update gifnoc
breuleux Apr 2, 2024
fadb95b
Merge branch 'master' into health-monitor
nurbal Apr 11, 2024
f890a18
Merge branch 'master' into health-monitor
nurbal Apr 16, 2024
cc2ec10
Merge branch 'master' into health-monitor
nurbal Apr 18, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3,369 changes: 1,768 additions & 1,601 deletions poetry.lock

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ nbsphinx = "^0.9.3"
opentelemetry-api = "^1.23.0"
opentelemetry-sdk = "^1.23.0"
requests-mock = "^1.11.0"
pandas = "< 2.1.0"
gifnoc = "^0.2.3"
watchdog = "^4.0.0"
python-dateutil = "^2.9.0.post0"

[tool.poetry.group.dev.dependencies]
black = ">= 22.12.0"
Expand Down
Empty file added sarc/alerts/__init__.py
Empty file.
100 changes: 100 additions & 0 deletions sarc/alerts/cache.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
"""Timespan-dependent cache."""

import inspect
from dataclasses import dataclass
from datetime import datetime, timedelta
from functools import cached_property, wraps
from typing import Optional

from apischema import deserialize
from gifnoc.std import time


@dataclass
class CachedResult:
"""Represents a result computed at some time."""

# Cached value
value: object = None
# Date at which the value was produced
issued: datetime = None


@dataclass(unsafe_hash=True)
class Timespan:

# Time duration
duration: timedelta

# Time offset between the end of the duration and time.now()
offset: Optional[timedelta] = timedelta(seconds=0)

# How long a cached result for this timespan is valid (defaults to same as duration)
validity: Optional[timedelta] = None

def __post_init__(self):
if isinstance(self.duration, str):
self.duration = deserialize(timedelta, self.duration)
if isinstance(self.offset, str):
self.offset = deserialize(timedelta, self.offset)
if isinstance(self.validity, str):
self.validity = deserialize(timedelta, self.validity)
elif self.validity is None:
self.validity = self.duration

def calculate_bounds(self, anchor=None):
"""Calculate time bounds (start, end).

The anchor is the end of the time period (default: time.now()). The offset
is subtracted from it to give `end`, and duration is subtracted from `end`
to give `start`.
"""
end = (anchor or time.now()) - self.offset
start = end - self.duration
return (start, end)

@cached_property
def bounds(self):
"""Return the time bounds (start, end) anchored at time.now()."""
return self.calculate_bounds()

@cached_property
def key(self):
"""Key for caching."""
# Validity does not need to be part of the key because the cached
# information does not depend on the validity period.
return (self.duration, self.offset)

def __str__(self):
s = f"{self.duration}"
if self.offset:
s += f" {self.offset} ago"
return s


def spancache(fn):
"""Decorator to cache a function's result for a certain duration.

The function's first argument should be a Timespan object which contains
a duration, optional offset, and a validity period.
"""
if "self" in inspect.signature(fn).parameters:
# It's just kind of a pain in the ass, we can try to make it work if
# necessary.
raise TypeError("@spancache does not work on methods")

cache = {}

@wraps(fn)
def wrapped(timespan, *args, **kwargs):
if current := cache.get(timespan.key, None):
if time.now() < current.issued + timespan.validity:
return current.value

value = fn(timespan, *args, **kwargs)
entry = CachedResult(value=value, issued=time.now())

cache[timespan.key] = entry
return value

return wrapped
Loading
Loading