kube-prometheus-stack

ben.wangzLess than 1 minute

kube-prometheus-stack

prepare

k8s is ready
argocd is ready and logged in
ingress is ready
cert-manager is ready
- the clusterissuer named self-signed-ca-issuer is ready

installation

prepare kube-prometheus-stack.yaml

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: kube-prometheus-stack
spec:
  syncPolicy:
    syncOptions:
    - CreateNamespace=true
    - ServerSideApply=true
  project: default
  source:
    repoURL: https://prometheus-community.github.io/helm-charts
    chart: kube-prometheus-stack
    targetRevision: 59.1.0
    helm:
      releaseName: kube-prometheus-stack
      values: |
        crds:
          enabled: true
        global:
          rbac:
            create: true
        alertmanager:
          enabled: true
          ingress:
            enabled: false
          serviceMonitor:
            selfMonitor: true
          alertmanagerSpec:
            image:
              registry: quay.io
            storage:
              volumeClaimTemplate:
                spec:
                  storageClassName: nfs-external
                  accessModes:
                  - ReadWriteOnce
                  resources:
                    requests:
                      storage: 20Gi
        grafana:
          enabled: true
          image:
            registry: docker.io
          testFramework:
            enabled: true
            image:
              registry: docker.io
          downloadDashboardsImage:
            registry: docker.io
          serviceMonitor:
            enabled: true
          ingress:
            enabled: true
            annotations:
              cert-manager.io/clusterissuer: self-signed-issuer
            ingressClassName: nginx
            path: /
            pathtype: ImplementationSpecific
            hosts:
            - grafana.dev.geekcity.tech
            tls:
            - secretName: grafana.dev.geekcity.tech-tls
              hosts:
              - grafana.dev.geekcity.tech
          persistence:
            enabled: true
            storageClassName: nfs-external
          initChownData:
            enabled: true
            image:
              registry: docker.io
          admin:
            existingSecret: kube-prometheus-stack-credentials
            userKey: grafana-username
            passwordKey: grafana-password
          datasources: {}
          dashboardProviders: {}
          dashboards: {}
          sidecar:
            image:
              registry: quay.io
            dashboards:
              enabled: true
            datasources:
              enabled: true
          imageRenderer:
            enabled: false
            image:
              registry: docker.io
        kubernetesServiceMonitors:
          enabled: true
          sidecar:
        kubeApiServer:
          enabled: true
        kubelet:
          enabled: true
          namespace: kube-system
        kubeControllerManager:
          enabled: true
          serviceMonitor:
            enabled: true
        coreDns:
          enabled: true
          serviceMonitor:
            enabled: true
        kubeDns:
          enabled: false
        kubeEtcd:
          enabled: true
          service:
            enabled: true
          serviceMonitor:
            enabled: true
        kubeScheduler:
          enabled: true
          service:
            enabled: true
          serviceMonitor:
            enabled: true
            insecureSkipVerify: true
        kubeProxy:
          enabled: true
          service:
            enabled: true
          serviceMonitor:
            enabled: true
        kubeStateMetrics:
          enabled: true
        kube-state-metrics:
          image:
            registry: registry.k8s.io
          prometheus:
            monitor:
              enabled: true
          selfMonitor:
            enabled: false
        nodeExporter:
          enabled: true
        prometheus-node-exporter:
          image:
            registry: quay.io
          prometheus:
            monitor:
              enabled: true
        prometheusOperator:
          enabled: true
          admissionWebhooks:
            enabled: true
            deployment:
              enabled: false
              image:
                registry: quay.io
            patch:
              enabled: true
              image:
                registry: registry.k8s.io
            certManager:
              enabled: false
          serviceAccount:
            create: true
          service:
            type: ClusterIP
          serviceMonitor:
            selfMonitor: true
        prometheus:
          enabled: true
          serviceAccount:
            create: true
          thanosService:
            enabled: false
          thanosServiceMonitor:
            enabled: false
          thanosServiceExternal:
            enabled: false
          service:
            type: ClusterIP
          servicePerReplica:
            enabled: false
          podDisruptionBudget:
            enabled: false
          thanosIngress:
            enabled: false
          ingress:
            enabled: true
            annotations:
              cert-manager.io/clusterissuer: self-signed-issuer
            ingressClassName: nginx
            paths:
            - /
            pathtype: ImplementationSpecific
            hosts:
            - prometheus.dev.geekcity.tech
            tls:
            - secretName: prometheus.dev.geekcity.tech-tls
              hosts:
              - prometheus.dev.geekcity.tech
          serviceMonitor:
            selfMonitor: true
          prometheusSpec:
            image:
              registry: quay.io
            storageSpec:
              volumeClaimTemplate:
                spec:
                  storageClassName: nfs-external
                  accessModes:
                  - ReadWriteOnce
                  resources:
                    requests:
                      storage: 20Gi
        thanosRuler:
          enabled: false
          ingress:
            enabled: false
          serviceMonitor:
            selfMonitor: true
          thanosRulerSpec:
            image:
              registry: quay.io
          storage:
            volumeClaimTemplate:
              spec:
                storageClassName: nfs-external
                accessModes:
                - ReadWriteOnce
                resources:
                  requests:
                    storage: 20Gi
        windowsMonitoring:
          enabled: false
  destination:
    server: https://kubernetes.default.svc
    namespace: monitor

prepare admin credentials secret

kubectl get namespaces monitor > /dev/null 2>&1 || kubectl create namespace monitor
kubectl -n monitor create secret generic kube-prometheus-stack-credentials \
    --from-literal=grafana-username=admin \
    --from-literal=grafana-password=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 16)

apply to k8s

kubectl -n argocd apply -f kube-prometheus-stack.yaml

sync by argocd

argocd app sync argocd/kube-prometheus-stack

NOTES
- there may be a bug: sync again will continuous to submit patch job
- there may be a bug: sync will failed but everything seems to work well

visit grafana

grafana.dev.geekcity.tech should be resolved to nginx-ingress
- for example, add $K8S_MASTER_IP grafana.dev.geekcity.tech to /etc/hosts

https://grafana.dev.geekcity.tech:32443

username

kubectl -n monitor get secret kube-prometheus-stack-credentials -o jsonpath='{.data.grafana-username}' | base64 -d

password

kubectl -n monitor get secret kube-prometheus-stack-credentials -o jsonpath='{.data.grafana-password}' | base64 -d

visit prometheus

prometheus.dev.geekcity.tech should be resolved to nginx-ingress
- for example, add $K8S_MASTER_IP prometheus.dev.geekcity.tech to /etc/hosts
https://prometheus.dev.geekcity.tech:32443

monitor services

mariadb

references

https://github.com/prometheus-operator/prometheus-operator
https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack
https://github.com/prometheus-operator/prometheus-operator/tree/main/Documentation/user-guides
https://github.com/prometheus/mysqld_exporter/tree/main/mysqld-mixin