add tests, added policies to scaleUp hpa rules, improved dashboard
This commit is contained in:
wboughattas
@@ -29,13 +29,17 @@ spec:
|
||||
name: nginx_http_requests_total
|
||||
target:
|
||||
type: AverageValue
|
||||
averageValue: "500" # Scale up if requests > 500 per second per pod
|
||||
averageValue: "500" # Scale up if requests per second > 500 per pod
|
||||
behavior:
|
||||
scaleDown:
|
||||
stabilizationWindowSeconds: 300 # Wait 5 minutes before removing a pod
|
||||
stabilizationWindowSeconds: 60 # 60 sec before removing a pod
|
||||
policies:
|
||||
- type: Percent
|
||||
value: 100
|
||||
periodSeconds: 15
|
||||
scaleUp:
|
||||
stabilizationWindowSeconds: 0 # Scale up immediately when busy
|
||||
stabilizationWindowSeconds: 60
|
||||
policies:
|
||||
- type: Pods
|
||||
value: 1
|
||||
periodSeconds: 60
|
||||
|
||||
@@ -590,7 +590,7 @@ data:
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"axisPlacement": "left",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
@@ -632,7 +632,8 @@ data:
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
@@ -644,6 +645,9 @@ data:
|
||||
{
|
||||
"id": "custom.axisPlacement",
|
||||
"value": "right"
|
||||
},
|
||||
{
|
||||
"id": "unit"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -689,7 +693,7 @@ data:
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": false,
|
||||
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"severed-apps\", pod=~\"severed-blog.*\", container!=\"\"}[1m])) / 0.1 * 100",
|
||||
"expr": "avg(rate(container_cpu_usage_seconds_total{namespace=\"severed-apps\", pod=~\"severed-blog.*\", container!=\"\"}[1m])) / 0.1 * 100",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"legendFormat": "CPU Saturation (100% Saturation = 90% Usage)",
|
||||
@@ -702,7 +706,7 @@ data:
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(container_memory_working_set_bytes{namespace=\"severed-apps\", pod=~\"severed-blog.*\", container!=\"\"}) / (100 * 1024 * 1024) * 100",
|
||||
"expr": "avg(container_memory_working_set_bytes{namespace=\"severed-apps\", pod=~\"severed-blog.*\", container!=\"\"}) / (100 * 1024 * 1024) * 100",
|
||||
"hide": false,
|
||||
"instant": false,
|
||||
"legendFormat": "Memory Saturation (100% Saturation = 80% Usage)",
|
||||
@@ -840,6 +844,82 @@ data:
|
||||
],
|
||||
"type": "table"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 10,
|
||||
"x": 14,
|
||||
"y": 22
|
||||
},
|
||||
"id": 12,
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": false
|
||||
},
|
||||
"maxVizHeight": 300,
|
||||
"minVizHeight": 16,
|
||||
"minVizWidth": 8,
|
||||
"namePlacement": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showUnfilled": true,
|
||||
"sizing": "auto",
|
||||
"valueMode": "color"
|
||||
},
|
||||
"pluginVersion": "12.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "time() - kube_pod_created{namespace=\"severed-apps\", pod=~\"severed-blog.*\"}",
|
||||
"legendFormat": "{{pod}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Pod Age",
|
||||
"type": "bargauge"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
@@ -909,8 +989,8 @@ data:
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 6,
|
||||
"w": 5,
|
||||
"h": 9,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 26
|
||||
},
|
||||
@@ -953,6 +1033,141 @@ data:
|
||||
],
|
||||
"title": "Traffic Quality (24h)",
|
||||
"type": "piechart"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 8,
|
||||
"x": 6,
|
||||
"y": 26
|
||||
},
|
||||
"id": 14,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"percentChangeColorMode": "standard",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "12.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "kube_deployment_status_replicas_unavailable{namespace=\"severed-apps\", deployment=\"severed-blog\"}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Deployment Health (best=0)",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 18,
|
||||
"x": 6,
|
||||
"y": 30
|
||||
},
|
||||
"id": 13,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"percentChangeColorMode": "standard",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "12.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (phase) (kube_pod_status_phase{namespace=\"severed-apps\", pod=~\"severed-blog.*\"})",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Pod Status",
|
||||
"type": "stat"
|
||||
}
|
||||
],
|
||||
"preload": false,
|
||||
@@ -970,5 +1185,5 @@ data:
|
||||
"timezone": "",
|
||||
"title": "Severed Cluster Health",
|
||||
"uid": "severed-health",
|
||||
"version": 1
|
||||
"version": 2
|
||||
}
|
||||
|
||||
@@ -49,6 +49,7 @@ kubectl logs -n monitoring -l name=alloy --tail=50
|
||||
[//]: # (kubectl logs -n severed-apps -l app=severed-blog -f)
|
||||
|
||||
[//]: # (kubectl logs loki-0 -n monitoring --tail=20)
|
||||
[//]: # (kubectl rollout restart deployment/grafana -n monitoring)
|
||||
|
||||
* **Internal Handshake:** Use your `access-hub.sh` script and visit `localhost:12345`.
|
||||
* Find the `prometheus.exporter.nginx.blog` component.
|
||||
|
||||
16
scripts/tests/generated-202-404-blog.sh
Normal file
16
scripts/tests/generated-202-404-blog.sh
Normal file
@@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
while true; do
|
||||
TIMESTAMP=$(date +"%Y-%m-%d %H:%M:%S")
|
||||
|
||||
# 1. Normal Request (200 OK)
|
||||
curl -s -o /dev/null "http://blog.localhost:8080/"
|
||||
echo "[$TIMESTAMP] Ping 200 OK"
|
||||
|
||||
# 2. Bad Request (404 Not Found)
|
||||
curl -s -o /dev/null "http://blog.localhost:8080/page-that-does-not-exist"
|
||||
echo "[$TIMESTAMP] Ping 404 Not Found"
|
||||
|
||||
done
|
||||
6
scripts/tests/stress-blog.sh
Normal file
6
scripts/tests/stress-blog.sh
Normal file
@@ -0,0 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
# 50 concurrent users, running forever
|
||||
ab -k -c 50 -t 300 -H "Host: blog.localhost" http://127.0.0.1:8080/
|
||||
Reference in New Issue
Block a user