add tests, added policies to scaleUp hpa rules, improved dashboard

apps/severed-blog-hpa.yaml

@@ -29,13 +29,17 @@ spec:
           name: nginx_http_requests_total
         target:
           type: AverageValue
-          averageValue: "500"         # Scale up if requests > 500 per second per pod
+          averageValue: "500"         # Scale up if requests per second > 500 per pod
   behavior:
     scaleDown:
-      stabilizationWindowSeconds: 300 # Wait 5 minutes before removing a pod
+      stabilizationWindowSeconds: 60 # 60 sec before removing a pod
       policies:
         - type: Percent
           value: 100
           periodSeconds: 15
     scaleUp:
-      stabilizationWindowSeconds: 0   # Scale up immediately when busy
+      stabilizationWindowSeconds: 60
+      policies:
+        - type: Pods
+          value: 1
+          periodSeconds: 60

infra/observer/dashboard-json.yaml

@@ -590,7 +590,7 @@ data:
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
                 "axisLabel": "",
-                "axisPlacement": "auto",
+                "axisPlacement": "left",
                 "barAlignment": 0,
                 "barWidthFactor": 0.6,
                 "drawStyle": "line",
@@ -632,7 +632,8 @@ data:
                     "value": 80
                   }
                 ]
-              }
+              },
+              "unit": "percent"
             },
             "overrides": [
               {
@@ -644,6 +645,9 @@ data:
                   {
                     "id": "custom.axisPlacement",
                     "value": "right"
+                  },
+                  {
+                    "id": "unit"
                   }
                 ]
               }
@@ -689,7 +693,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": false,
-              "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"severed-apps\", pod=~\"severed-blog.*\", container!=\"\"}[1m])) / 0.1 * 100",
+              "expr": "avg(rate(container_cpu_usage_seconds_total{namespace=\"severed-apps\", pod=~\"severed-blog.*\", container!=\"\"}[1m])) / 0.1 * 100",
               "hide": false,
               "instant": false,
               "legendFormat": "CPU Saturation (100% Saturation = 90% Usage)",
@@ -702,7 +706,7 @@ data:
                 "uid": "PBFA97CFB590B2093"
               },
               "editorMode": "code",
-              "expr": "sum(container_memory_working_set_bytes{namespace=\"severed-apps\", pod=~\"severed-blog.*\", container!=\"\"}) / (100 * 1024 * 1024) * 100",
+              "expr": "avg(container_memory_working_set_bytes{namespace=\"severed-apps\", pod=~\"severed-blog.*\", container!=\"\"}) / (100 * 1024 * 1024) * 100",
               "hide": false,
               "instant": false,
               "legendFormat": "Memory Saturation (100% Saturation = 80% Usage)",
@@ -840,6 +844,82 @@ data:
           ],
           "type": "table"
         },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 10,
+            "x": 14,
+            "y": 22
+          },
+          "id": 12,
+          "options": {
+            "displayMode": "gradient",
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": false
+            },
+            "maxVizHeight": 300,
+            "minVizHeight": 16,
+            "minVizWidth": 8,
+            "namePlacement": "auto",
+            "orientation": "auto",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "showUnfilled": true,
+            "sizing": "auto",
+            "valueMode": "color"
+          },
+          "pluginVersion": "12.3.1",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "time() - kube_pod_created{namespace=\"severed-apps\", pod=~\"severed-blog.*\"}",
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Pod Age",
+          "type": "bargauge"
+        },
         {
           "datasource": {
             "type": "loki",
@@ -909,8 +989,8 @@ data:
             ]
           },
           "gridPos": {
-            "h": 6,
+            "h": 9,
-            "w": 5,
+            "w": 6,
             "x": 0,
             "y": 26
           },
@@ -953,6 +1033,141 @@ data:
           ],
           "title": "Traffic Quality (24h)",
           "type": "piechart"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 4,
+            "w": 8,
+            "x": 6,
+            "y": 26
+          },
+          "id": 14,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "percentChangeColorMode": "standard",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "showPercentChange": false,
+            "textMode": "auto",
+            "wideLayout": true
+          },
+          "pluginVersion": "12.3.1",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "kube_deployment_status_replicas_unavailable{namespace=\"severed-apps\", deployment=\"severed-blog\"}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Deployment Health (best=0)",
+          "type": "stat"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": 0
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 5,
+            "w": 18,
+            "x": 6,
+            "y": 30
+          },
+          "id": 13,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "auto",
+            "percentChangeColorMode": "standard",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "showPercentChange": false,
+            "textMode": "auto",
+            "wideLayout": true
+          },
+          "pluginVersion": "12.3.1",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "sum by (phase) (kube_pod_status_phase{namespace=\"severed-apps\", pod=~\"severed-blog.*\"})",
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Pod Status",
+          "type": "stat"
         }
       ],
       "preload": false,
@@ -970,5 +1185,5 @@ data:
       "timezone": "",
       "title": "Severed Cluster Health",
       "uid": "severed-health",
-      "version": 1
+      "version": 2
     }

scripts/README.md

@@ -49,6 +49,7 @@ kubectl logs -n monitoring -l name=alloy --tail=50
 [//]: # (kubectl logs -n severed-apps -l app=severed-blog -f)
 
 [//]: # (kubectl logs loki-0 -n monitoring --tail=20)
+[//]: # (kubectl rollout restart deployment/grafana -n monitoring)
 
 * **Internal Handshake:** Use your `access-hub.sh` script and visit `localhost:12345`.
 * Find the `prometheus.exporter.nginx.blog` component.

scripts/tests/generated-202-404-blog.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e
+
+while true; do
+  TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S")
+
+  # 1. Normal Request (200 OK)
+  curl -s -o /dev/null "http://blog.localhost:8080/"
+  echo "[$TIMESTAMP] Ping 200 OK"
+
+  # 2. Bad Request (404 Not Found)
+  curl -s -o /dev/null "http://blog.localhost:8080/page-that-does-not-exist"
+  echo "[$TIMESTAMP] Ping 404 Not Found"
+
+done

scripts/tests/stress-blog.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+
+# 50 concurrent users, running forever
+ab -k -c 50 -t 300 -H "Host: blog.localhost" http://127.0.0.1:8080/