k8s - otel, prometheus, kubelet metrics, jeager

giovannymassuia · Dec 1, 2024 · e589870 · e589870
1 parent 7f9b5b2
commit e589870
Show file tree

Hide file tree

Showing 8 changed files with 241 additions and 33 deletions.
diff --git a/...ct-catalog/src/main/java/io/giovannymassuia/disecomm/productcatalog/utils/ChaosUtils.java b/...ct-catalog/src/main/java/io/giovannymassuia/disecomm/productcatalog/utils/ChaosUtils.java
@@ -56,12 +56,22 @@ public static void applyChaos(boolean randomize) {
         }
 
         System.out.println("Applying chaos...");
+        throwRandomException();
         addRandomDelay();
         stressCPUWithFibonacci();
         stressMemory();
         System.out.println("Chaos applied!");
     }
 
+    /**
+     * Helper method to throw a random exception.
+     */
+    private static void throwRandomException() {
+        if (RANDOM.nextBoolean()) {
+            throw new RuntimeException("Random exception thrown by chaos monkey!");
+        }
+    }
+
     /**
      * Helper method to calculate Fibonacci recursively.
      */

diff --git a/local-k8s/helm/otel/notes.md b/local-k8s/helm/otel/notes.md
@@ -15,14 +15,36 @@ helm install otel-collector open-telemetry/opentelemetry-collector
 
 1. Install Grafana
    - https://grafana.com/docs/grafana/latest/setup-grafana/installation/helm/ 
+     - ```bash
+         helm install grafana grafana/grafana \
+          --namespace monitoring \
+          --values local-k8s/monitoring/grafana/helm-values.yaml
+       ```
 2. Install Prometheus
+    - `helm install ksm prometheus-community/kube-state-metrics --namespace monitoring`
 3. Install Loki
 4. Install Tempo
 5. Install OpenTelemetry Operator
    - pre-requisite: cert-manager
      - https://cert-manager.io/docs/installation/helm/
+       - ```bash 
+           helm install \
+            cert-manager jetstack/cert-manager \
+            --namespace cert-manager \
+            --create-namespace \
+            --version v1.16.2 \
+            --set crds.enabled=true
+         ```
    - https://opentelemetry.io/docs/kubernetes/operator/
    - https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-operator
+       - ```bash 
+            helm install opentelemetry-operator open-telemetry/opentelemetry-operator \
+            --namespace monitoring \
+            --create-namespace \
+             --set "manager.collectorImage.repository=otel/opentelemetry-collector-k8s" \
+             --set admissionWebhooks.certManager.enabled=false \
+             --set admissionWebhooks.autoGenerateCert.enabled=true 
+         ```
 6. Install OpenTelemetry Collector
 
 ```bash

diff --git a/local-k8s/monitoring/jaeger/deployment.yaml b/local-k8s/monitoring/jaeger/deployment.yaml
@@ -0,0 +1,61 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: jaeger
+  namespace: monitoring
+spec:
+  selector:
+    matchLabels:
+      app: jaeger
+  template:
+    metadata:
+      name: jaeger
+      labels:
+        app: jaeger
+    spec:
+      containers:
+        - name: app
+          image: jaegertracing/all-in-one:1.60.0
+          ports:
+            - containerPort: 16686
+              name: ui
+            - containerPort: 4317
+              name: otlp-grpc
+          env:
+            - name: COLLECTOR_OTLP_ENABLED
+              value: "true"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: jaeger
+  namespace: monitoring
+spec:
+  selector:
+    app: jaeger
+  ports:
+    - port: 16686
+      name: ui
+    - port: 4317
+      name: otlp-grpc
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: jaeger-ui-ingress
+  namespace: monitoring
+  annotations:
+    traefik.ingress.kubernetes.io/router.entrypoints: web
+spec:
+  ingressClassName: traefik
+  rules:
+    - host: "jeager.127.0.0.1.nip.io"
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: jaeger
+                port:
+                  number: 16686
diff --git a/local-k8s/monitoring/otel/otel-auto-instrumentation.yaml b/local-k8s/monitoring/otel/otel-auto-instrumentation.yaml
@@ -2,6 +2,7 @@ apiVersion: opentelemetry.io/v1alpha1
 kind: Instrumentation
 metadata:
   name: java-instrumentation
+  namespace: disecomm-apps
 spec:
   exporter:
     endpoint: http://simplest-collector.monitoring:4317

diff --git a/local-k8s/monitoring/otel/otel-collector.yaml b/local-k8s/monitoring/otel/otel-collector.yaml
@@ -40,31 +40,51 @@ spec:
       debug:
         verbosity: detailed
       # https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlphttpexporter
-      otlphttp:
-        logs_endpoint: http://loki:3100/otlp/v1/logs
-        metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics
+      # https://grafana.com/docs/loki/latest/reference/loki-http-api/
+
+      #    OTLP/HTTP exporter option 1
+      # otlphttp:
+      #   logs_endpoint: http://loki:3100/otlp/v1/logs
+      #   metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics
+      #   tls:
+      #     insecure: true
+
+      #    OTLP/HTTP exporter option 2
+      otlphttp/prometheus: # '/prometheus' is my custom name for this exporter
+        endpoint: http://prometheus:9090/api/v1/otlp
         tls:
           insecure: true
-#      otlphttp/logs:
-#        endpoint: http://loki:3100/otlp
-#        tls:
-#          insecure: true
-      otlp:
+      otlphttp/loki: # '/logs' is my custom name for this exporter
+        endpoint: http://loki:3100/otlp
+        tls:
+          insecure: true
+
+      otlp/my-tempo:
         endpoint: tempo:4317
         tls:
           insecure: true
 
+      otlp/oteltui:
+        endpoint: otel-tui:4317
+        tls:
+          insecure: true
+
+      otlp/jaeger:
+        endpoint: jaeger:4317
+        tls:
+          insecure: true
+
     service:
       pipelines:
         logs:
           receivers: [otlp]
-          processors: [ resource, attributes, batch ]
-          exporters: [debug, otlphttp]
+          processors: [resource, attributes, batch]
+          exporters: [debug, otlphttp/loki, otlp/oteltui]
         metrics:
           receivers: [otlp]
-          processors: [ resource, batch ]
-          exporters: [debug, otlphttp]
+          processors: [resource, batch]
+          exporters: [debug, otlphttp/prometheus, otlp/oteltui]
         traces:
           receivers: [otlp]
-          processors: [ resource, batch ]
-          exporters: [debug, otlp]
+          processors: [resource, batch]
+          exporters: [debug, otlp/my-tempo, otlp/oteltui, otlp/jaeger]
diff --git a/local-k8s/monitoring/otel/otel-tui.yaml b/local-k8s/monitoring/otel/otel-tui.yaml
@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: otel-tui
+  namespace: monitoring
+spec:
+  selector:
+    matchLabels:
+      app: otel-tui
+  template:
+    metadata:
+      name: otel-tui
+      labels:
+        app: otel-tui
+    spec:
+      containers:
+        - name: app
+          image: ymtdzzz/otel-tui:v0.3.7
+          stdin: true
+          tty: true
+          ports:
+            - containerPort: 4317
+              name: grpc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: otel-tui
+  namespace: monitoring
+spec:
+  selector:
+    app: otel-tui
+  ports:
+    - port: 4317
+      name: grpc
diff --git a/local-k8s/monitoring/prometheus/prometheus-statefulset.yaml b/local-k8s/monitoring/prometheus/prometheus-statefulset.yaml
@@ -1,21 +1,46 @@
 apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus
+  namespace: monitoring
+---
+apiVersion: v1
 kind: ConfigMap
 metadata:
   name: prometheus-config
   namespace: monitoring
 data:
-    prometheus.yml: |
-      global:
-        scrape_interval: 10s
-        evaluation_interval: 10s
+  prometheus.yml: |
+    global:
+      scrape_interval: 10s
+      evaluation_interval: 10s
 
-      scrape_configs:
-        # - job_name: 'otel-collector'
-        #   static_configs:
-        #     - targets: [ 'simplest-collector:8889' ]
-        - job_name: 'kube-state-metrics'
-          static_configs:
-            - targets: [ 'ksm-kube-state-metrics:8080' ]
+    scrape_configs:
+      # - job_name: 'otel-collector'
+      #   static_configs:
+      #     - targets: [ 'simplest-collector:8889' ]
+      - job_name: 'kube-state-metrics'
+        static_configs:
+          - targets: [ 'ksm-kube-state-metrics:8080' ]
+      - job_name: 'kubelet'
+        scheme: https
+        tls_config:
+          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+        kubernetes_sd_configs:
+          - role: node
+        relabel_configs:
+          - action: labelmap
+            regex: __meta_kubernetes_node_label_(.+)
+          - source_labels: [__meta_kubernetes_node_name]
+            action: replace
+            target_label: node
+          - target_label: __address__
+            replacement: kubernetes.default.svc:443
+          - source_labels: [__meta_kubernetes_node_name]
+            regex: (.+)
+            target_label: __metrics_path__
+            replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -36,6 +61,7 @@ spec:
       labels:
         db: prometheus
     spec:
+      serviceAccountName: prometheus
       containers:
         - name: prometheus
           image: prom/prometheus:v3.0.1
@@ -45,15 +71,15 @@ spec:
             - containerPort: 8889
               name: remote-write
           args:
-            - '--config.file=/etc/prometheus/prometheus.yml'
-            - '--storage.tsdb.path=/prometheus'
-            - '--storage.tsdb.retention.time=1h'
-            - '--web.enable-remote-write-receiver' # This is necessary to k6 to send metrics to Prometheus
-            - '--enable-feature=native-histograms'
-            - '--web.enable-otlp-receiver'
+            - "--config.file=/etc/prometheus/prometheus.yml"
+            - "--storage.tsdb.path=/prometheus"
+            - "--storage.tsdb.retention.time=1h"
+            - "--web.enable-remote-write-receiver" # This is necessary to k6 to send metrics to Prometheus
+            - "--enable-feature=native-histograms"
+            - "--web.enable-otlp-receiver"
           volumeMounts:
-              - mountPath: /etc/prometheus
-                name: prometheus-config
+            - mountPath: /etc/prometheus
+              name: prometheus-config
       volumes:
         - name: prometheus-config
           configMap:

diff --git a/local-k8s/monitoring/prometheus/rbac.yaml b/local-k8s/monitoring/prometheus/rbac.yaml
@@ -0,0 +1,33 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus-kubelet
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes/metrics
+      - nodes/proxy
+      - nodes/stats
+      - nodes/spec
+      - nodes
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources:
+      - endpoints
+      - services
+      - pods
+    verbs: ["get", "list", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus-kubelet
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus-kubelet
+subjects:
+  - kind: ServiceAccount
+    name: prometheus
+    namespace: monitoring