Severed-Infra/infra/alloy-setup.yaml

# --- RBAC configuration ---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: alloy-sa
  namespace: monitoring

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: alloy-cluster-role
rules:
  # 1. Standard API Access
  - apiGroups: [ "" ]
    resources: [ "nodes", "nodes/proxy", "services", "endpoints", "pods" ]
    verbs: [ "get", "list", "watch" ]
  # 2. ALLOW METRICS ACCESS (Crucial for cAdvisor/Kubelet)
  - apiGroups: [ "" ]
    resources: [ "nodes/stats", "nodes/metrics" ]
    verbs: [ "get" ]
  # 3. Log Access
  - apiGroups: [ "" ]
    resources: [ "pods/log" ]
    verbs: [ "get", "list", "watch" ]
  # 4. Non-Resource URLs (Sometimes needed for /metrics endpoints)
  - nonResourceURLs: ["/metrics"]
    verbs: ["get"]

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: alloy-cluster-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: alloy-cluster-role
subjects:
  - kind: ServiceAccount
    name: alloy-sa
    namespace: monitoring

---
# --- Alloy pipeline configuration ---
apiVersion: v1
kind: ConfigMap
metadata:
  name: alloy-config
  namespace: monitoring
data:
  config.alloy: |
    // 1. Discovery: Find all pods
    discovery.kubernetes "k8s_pods" {
      role = "pod"
    }

    // 2. Relabeling: Filter and Label "severed-blog" pods
    discovery.relabel "blog_pods" {
      targets = discovery.kubernetes.k8s_pods.targets
    
      rule {
        action = "keep"
        source_labels = ["__meta_kubernetes_pod_label_app"]
        regex = "severed-blog"
      }

      // Explicitly set 'pod' and 'namespace' labels for the Adapter
      rule {
        action = "replace"
        source_labels = ["__meta_kubernetes_pod_name"]
        target_label = "pod"
      }

      rule {
        action = "replace"
        source_labels = ["__meta_kubernetes_namespace"]
        target_label = "namespace"
      }

      // Route to the sidecar exporter port
      rule {
        action = "replace"
        source_labels = ["__address__"]
        target_label = "__address__"
        regex = "([^:]+)(?::\\d+)?"
        replacement = "$1:9113"
      }
    }

    // 3. Direct Nginx Scraper
    prometheus.scrape "nginx_scraper" {
      targets = discovery.relabel.blog_pods.output
      forward_to = [prometheus.remote_write.metrics_service.receiver]
      job_name   = "integrations/nginx"
    }

    // 4. Host Metrics (Unix Exporter)
    prometheus.exporter.unix "host" {
      rootfs_path = "/host/root"
      sysfs_path  = "/host/sys"
      procfs_path = "/host/proc"
    }

    prometheus.scrape "host_scraper" {
      targets    = prometheus.exporter.unix.host.targets
      forward_to = [prometheus.remote_write.metrics_service.receiver]
    }

    // 5. Remote Write: Send to Prometheus
    prometheus.remote_write "metrics_service" {
      endpoint {
        url = sys.env("PROM_URL")
      }
    }

    // 6. Logs Pipeline: Send to Loki
    loki.source.kubernetes "pod_logs" {
      targets    = discovery.relabel.blog_pods.output
      forward_to = [loki.write.default.receiver]
    }

    loki.write "default" {
      endpoint {
        url = sys.env("LOKI_URL")
      }
    }
    
    // 7. Kubelet Scraper (cAdvisor for Container Metrics)
    discovery.kubernetes "k8s_nodes" {
      role = "node"
    }

    prometheus.scrape "kubelet_cadvisor" {
      targets = discovery.kubernetes.k8s_nodes.targets
      scheme  = "https"
      metrics_path = "/metrics/cadvisor"
      job_name     = "integrations/kubernetes/cadvisor"

      tls_config {
        insecure_skip_verify = true
      }
      bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"

      forward_to = [prometheus.remote_write.metrics_service.receiver]
    }

---
# --- Agent Deployment (DaemonSet) ---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: alloy
  namespace: monitoring
spec:
  selector:
    matchLabels:
      name: alloy
  template:
    metadata:
      labels:
        name: alloy
    spec:
      serviceAccountName: alloy-sa
      hostNetwork: true
      hostPID: true
      dnsPolicy: ClusterFirstWithHostNet
      containers:
        - name: alloy
          image: grafana/alloy:latest
          args:
            - run
            - --server.http.listen-addr=0.0.0.0:12345
            - --storage.path=/var/lib/alloy/data
            - /etc/alloy/config.alloy
          envFrom:
            - configMapRef:
                name: monitoring-env
                optional: false
          volumeMounts:
            - name: config
              mountPath: /etc/alloy
            - name: logs
              mountPath: /var/log
            - name: proc
              mountPath: /host/proc
              readOnly: true
            - name: sys
              mountPath: /host/sys
              readOnly: true
            - name: root
              mountPath: /host/root
              readOnly: true
      volumes:
        - name: config
          configMap:
            name: alloy-config
        - name: logs
          hostPath:
            path: /var/log
        - name: proc
          hostPath:
            path: /proc
        - name: sys
          hostPath:
            path: /sys
        - name: root
          hostPath:
            path: /