From d0af234e7b18757a5890f0495e7aa27cd58bde7d Mon Sep 17 00:00:00 2001 From: wboughattas Date: Tue, 30 Dec 2025 21:49:46 -0500 Subject: [PATCH] add tests, added policies to scaleUp hpa rules, improved dashboard --- apps/severed-blog-hpa.yaml | 10 +- infra/observer/dashboard-json.yaml | 229 +++++++++++++++++++++++- scripts/README.md | 1 + scripts/tests/generated-202-404-blog.sh | 16 ++ scripts/tests/stress-blog.sh | 6 + 5 files changed, 252 insertions(+), 10 deletions(-) create mode 100644 scripts/tests/generated-202-404-blog.sh create mode 100644 scripts/tests/stress-blog.sh diff --git a/apps/severed-blog-hpa.yaml b/apps/severed-blog-hpa.yaml index 6559975..e10a6ce 100644 --- a/apps/severed-blog-hpa.yaml +++ b/apps/severed-blog-hpa.yaml @@ -29,13 +29,17 @@ spec: name: nginx_http_requests_total target: type: AverageValue - averageValue: "500" # Scale up if requests > 500 per second per pod + averageValue: "500" # Scale up if requests per second > 500 per pod behavior: scaleDown: - stabilizationWindowSeconds: 300 # Wait 5 minutes before removing a pod + stabilizationWindowSeconds: 60 # 60 sec before removing a pod policies: - type: Percent value: 100 periodSeconds: 15 scaleUp: - stabilizationWindowSeconds: 0 # Scale up immediately when busy + stabilizationWindowSeconds: 60 + policies: + - type: Pods + value: 1 + periodSeconds: 60 diff --git a/infra/observer/dashboard-json.yaml b/infra/observer/dashboard-json.yaml index b11bd28..c53dac2 100644 --- a/infra/observer/dashboard-json.yaml +++ b/infra/observer/dashboard-json.yaml @@ -590,7 +590,7 @@ data: "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", - "axisPlacement": "auto", + "axisPlacement": "left", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -632,7 +632,8 @@ data: "value": 80 } ] - } + }, + "unit": "percent" }, "overrides": [ { @@ -644,6 +645,9 @@ data: { "id": "custom.axisPlacement", "value": "right" + }, + { + "id": "unit" } ] } @@ -689,7 +693,7 @@ data: }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"severed-apps\", pod=~\"severed-blog.*\", container!=\"\"}[1m])) / 0.1 * 100", + "expr": "avg(rate(container_cpu_usage_seconds_total{namespace=\"severed-apps\", pod=~\"severed-blog.*\", container!=\"\"}[1m])) / 0.1 * 100", "hide": false, "instant": false, "legendFormat": "CPU Saturation (100% Saturation = 90% Usage)", @@ -702,7 +706,7 @@ data: "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(container_memory_working_set_bytes{namespace=\"severed-apps\", pod=~\"severed-blog.*\", container!=\"\"}) / (100 * 1024 * 1024) * 100", + "expr": "avg(container_memory_working_set_bytes{namespace=\"severed-apps\", pod=~\"severed-blog.*\", container!=\"\"}) / (100 * 1024 * 1024) * 100", "hide": false, "instant": false, "legendFormat": "Memory Saturation (100% Saturation = 80% Usage)", @@ -840,6 +844,82 @@ data: ], "type": "table" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 22 + }, + "id": 12, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "time() - kube_pod_created{namespace=\"severed-apps\", pod=~\"severed-blog.*\"}", + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Pod Age", + "type": "bargauge" + }, { "datasource": { "type": "loki", @@ -909,8 +989,8 @@ data: ] }, "gridPos": { - "h": 6, - "w": 5, + "h": 9, + "w": 6, "x": 0, "y": 26 }, @@ -953,6 +1033,141 @@ data: ], "title": "Traffic Quality (24h)", "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 6, + "y": 26 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "kube_deployment_status_replicas_unavailable{namespace=\"severed-apps\", deployment=\"severed-blog\"}", + "range": true, + "refId": "A" + } + ], + "title": "Deployment Health (best=0)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 18, + "x": 6, + "y": 30 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (phase) (kube_pod_status_phase{namespace=\"severed-apps\", pod=~\"severed-blog.*\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" } ], "preload": false, @@ -970,5 +1185,5 @@ data: "timezone": "", "title": "Severed Cluster Health", "uid": "severed-health", - "version": 1 + "version": 2 } diff --git a/scripts/README.md b/scripts/README.md index f9b3281..0502634 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -49,6 +49,7 @@ kubectl logs -n monitoring -l name=alloy --tail=50 [//]: # (kubectl logs -n severed-apps -l app=severed-blog -f) [//]: # (kubectl logs loki-0 -n monitoring --tail=20) +[//]: # (kubectl rollout restart deployment/grafana -n monitoring) * **Internal Handshake:** Use your `access-hub.sh` script and visit `localhost:12345`. * Find the `prometheus.exporter.nginx.blog` component. diff --git a/scripts/tests/generated-202-404-blog.sh b/scripts/tests/generated-202-404-blog.sh new file mode 100644 index 0000000..cf4fa2b --- /dev/null +++ b/scripts/tests/generated-202-404-blog.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +while true; do + TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S") + + # 1. Normal Request (200 OK) + curl -s -o /dev/null "http://blog.localhost:8080/" + echo "[$TIMESTAMP] Ping 200 OK" + + # 2. Bad Request (404 Not Found) + curl -s -o /dev/null "http://blog.localhost:8080/page-that-does-not-exist" + echo "[$TIMESTAMP] Ping 404 Not Found" + +done \ No newline at end of file diff --git a/scripts/tests/stress-blog.sh b/scripts/tests/stress-blog.sh new file mode 100644 index 0000000..45b89ef --- /dev/null +++ b/scripts/tests/stress-blog.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e + +# 50 concurrent users, running forever +ab -k -c 50 -t 300 -H "Host: blog.localhost" http://127.0.0.1:8080/