Deploy kube-prometheus-stack and Write PromQL Queries

# Add the Prometheus community Helm chart
helm repo add prometheus-community \
  https://prometheus-community.github.io/helm-charts
helm repo update

# Create a values.yaml for minimal but functional setup
cat > prom-values.yaml << 'EOF'
prometheus:
  prometheusSpec:
    retention: 7d
    storageSpec:
      volumeClaimTemplate:
        spec:
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 20Gi
grafana:
  enabled: true
  adminPassword: "admin123"
  persistence:
    enabled: true
    size: 5Gi
alertmanager:
  alertmanagerSpec:
    storage:
      volumeClaimTemplate:
        spec:
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 5Gi
EOF

helm install kube-prometheus-stack \
  prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --create-namespace \
  --values prom-values.yaml \
  --wait --timeout 10m

kubectl get pods -n monitoring

# Port-forward Prometheus
kubectl port-forward svc/kube-prometheus-stack-prometheus \
  -n monitoring 9090:9090 &

# Port-forward Grafana
kubectl port-forward svc/kube-prometheus-stack-grafana \
  -n monitoring 3000:80 &

# Access URLs:
# Prometheus: http://localhost:9090
# Grafana:    http://localhost:3000  (admin / admin123)

# Verify targets are being scraped
# In Prometheus UI: Status → Targets → should show all UP

# Node memory utilisation percentage (used / total)
(
  node_memory_MemTotal_bytes
  - node_memory_MemFree_bytes
  - node_memory_Buffers_bytes
  - node_memory_Cached_bytes
)
/ node_memory_MemTotal_bytes * 100

# Alert threshold: fire when any node exceeds 85%
# In Prometheus: Status → Rules to see pre-built node alerts

# Alternative: use kube_node_status_condition for pressure state
kube_node_status_condition{condition="MemoryPressure",status="true"}
# Value of 1 = node IS under memory pressure

# Count of pods in Pending state
count by (namespace) (
  kube_pod_status_phase{phase="Pending"}
)

# More useful: pending pods grouped by reason
kube_pod_status_unschedulable == 1

# Pending pods that have been pending for over 5 minutes
# (potential stuck scheduling)
(
  kube_pod_status_phase{phase="Pending"}
  and on(pod, namespace)
  (time() - kube_pod_created) > 300
)

# API server request rate by verb and resource (5-minute window)
sum by (verb, resource) (
  rate(apiserver_request_total[5m])
)

# API server error rate (4xx and 5xx)
sum by (code, verb) (
  rate(apiserver_request_total{code=~"[45].."}[5m])
)

# API server request latency P99 by verb
histogram_quantile(0.99,
  sum by (verb, le) (
    rate(apiserver_request_duration_seconds_bucket[5m])
  )
)

# PVCs not in Bound state
kube_persistentvolumeclaim_status_phase{phase!="Bound"}

# PVCs that have been Pending for over 10 minutes
(
  kube_persistentvolumeclaim_status_phase{phase="Pending"}
  and on(persistentvolumeclaim, namespace)
  (time() - kube_persistentvolumeclaim_created) > 600
)

# Count of unbound PVCs by namespace
count by (namespace) (
  kube_persistentvolumeclaim_status_phase{phase!="Bound"}
)

# Container restart rate in the last hour by namespace
sum by (namespace) (
  increase(kube_pod_container_status_restarts_total[1h])
)

# Top 10 most restarting containers
topk(10,
  sum by (namespace, pod, container) (
    increase(kube_pod_container_status_restarts_total[1h])
  )
)

# Alert expression: namespace with >10 restarts in last hour
sum by (namespace) (
  increase(kube_pod_container_status_restarts_total[1h])
) > 10

# Navigate to Grafana at http://localhost:3000
# Key dashboards installed by kube-prometheus-stack:

# Dashboard ID 315: Kubernetes cluster monitoring
# Dashboard ID 6417: Kubernetes pods
# Dashboard ID 1860: Node exporter full

# In Grafana: Dashboards → Browse → Kubernetes
# Try: "Kubernetes / Compute Resources / Cluster"
# This shows all 5 of our queries as panels

# View currently firing alerts
kubectl port-forward svc/kube-prometheus-stack-alertmanager \
  -n monitoring 9093:9093 &
# Open http://localhost:9093