From f551e75f978b16887b60afdb9a565d4105632830 Mon Sep 17 00:00:00 2001
From: YuviPanda <yuvipanda@gmail.com>
Date: Wed, 6 Sep 2023 15:11:06 -0700
Subject: [PATCH] Switch filestore alerts back to % based alerts

Requiring 512GiB to be free on a 1TiB filestore is too much.
So instead we require 10% to be free on all of them.
---
 terraform/gcp/pagerduty.tf | 8 ++++----
 terraform/gcp/variables.tf | 9 +++++++++
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/terraform/gcp/pagerduty.tf b/terraform/gcp/pagerduty.tf
index 5b4529068..7b79f3c01 100644
--- a/terraform/gcp/pagerduty.tf
+++ b/terraform/gcp/pagerduty.tf
@@ -26,7 +26,7 @@ resource "google_monitoring_notification_channel" "pagerduty_disk_space" {
 
 resource "google_monitoring_alert_policy" "disk_space_full_alert" {
 
-  display_name = "Disk Space 80% full on ${var.project_id}"
+  display_name = "Available disk space < ${var.filestore_alert_available_percent}% on ${var.project_id}"
   combiner     = "OR"
 
   conditions {
@@ -35,12 +35,12 @@ resource "google_monitoring_alert_policy" "disk_space_full_alert" {
       # Alert based on free bytes left on the filesystem
       filter   = <<-EOT
       resource.type = "filestore_instance"
-      AND metric.type = "file.googleapis.com/nfs/server/free_bytes"
+      AND metric.type = "file.googleapis.com/nfs/server/free_bytes_percent"
       EOT
       duration = "300s"
 
-      # Trigger if free space is < 512GB
-      threshold_value = 549755813888
+      # Trigger if free space is < 10%
+      threshold_value = var.filestore_alert_available_percent
       comparison      = "COMPARISON_LT"
 
       aggregations {
diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf
index 45cb956fe..76613e148 100644
--- a/terraform/gcp/variables.tf
+++ b/terraform/gcp/variables.tf
@@ -309,6 +309,15 @@ variable "filestore_tier" {
   EOT
 }
 
+variable "filestore_alert_available_percent" {
+  type        = number
+  default     = 10
+  description = <<-EOT
+  % of free space in filestore available under which to fire an alert to pagerduty.
+  EOT
+}
+
+
 variable "enable_node_autoprovisioning" {
   type        = bool
   default     = false