From ee75979b5d94ae18f930ff91e5b2d51cd554b60d Mon Sep 17 00:00:00 2001 From: Paul Buetow Date: Mon, 9 Mar 2026 22:45:54 +0200 Subject: Update content for html --- ...5-12-07-f3s-kubernetes-with-freebsd-part-8.html | 403 ++++++++++----------- 1 file changed, 199 insertions(+), 204 deletions(-) (limited to 'gemfeed/2025-12-07-f3s-kubernetes-with-freebsd-part-8.html') diff --git a/gemfeed/2025-12-07-f3s-kubernetes-with-freebsd-part-8.html b/gemfeed/2025-12-07-f3s-kubernetes-with-freebsd-part-8.html index 0cda1b53..f5280913 100644 --- a/gemfeed/2025-12-07-f3s-kubernetes-with-freebsd-part-8.html +++ b/gemfeed/2025-12-07-f3s-kubernetes-with-freebsd-part-8.html @@ -2,17 +2,12 @@ - f3s: Kubernetes with FreeBSD - Part 8: Observability -
-
-
-

Home | Markdown | Gemini

@@ -125,10 +120,10 @@ by Lorenzo Bettini http://www.lorenzobettini.it http://www.gnu.org/software/src-highlite --> -
$ git clone https://codeberg.org/snonux/conf.git
-$ cd conf
-$ git checkout 15a86f3  # Last commit before ArgoCD migration
-$ cd f3s/prometheus/
+
$ git clone https://codeberg.org/snonux/conf.git
+$ cd conf
+$ git checkout 15a86f3  # Last commit before ArgoCD migration
+$ cd f3s/prometheus/
 

 

 **Current master branch** contains the ArgoCD-managed versions with:

@@ -165,8 +160,8 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
$ kubectl create namespace monitoring
-namespace/monitoring created
+
$ kubectl create namespace monitoring
+namespace/monitoring created
 

 

 
Installing Prometheus and Grafana


@@ -181,8 +176,8 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
$ helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
-$ helm repo update
+
$ helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+$ helm repo update
 

 

 Create the directories on the NFS server for persistent storage:

@@ -191,8 +186,8 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
[root@r0 ~]# mkdir -p /data/nfs/k3svolumes/prometheus/data
-[root@r0 ~]# mkdir -p /data/nfs/k3svolumes/grafana/data
+
[root@r0 ~]# mkdir -p /data/nfs/k3svolumes/prometheus/data
+[root@r0 ~]# mkdir -p /data/nfs/k3svolumes/grafana/data
 

 

 
Deploying with the Justfile


@@ -208,18 +203,18 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
$ cd conf/f3s/prometheus
-$ just install
-kubectl apply -f persistent-volumes.yaml
-persistentvolume/prometheus-data-pv created
-persistentvolume/grafana-data-pv created
-persistentvolumeclaim/grafana-data-pvc created
-helm install prometheus prometheus-community/kube-prometheus-stack \
-    --namespace monitoring -f persistence-values.yaml
-NAME: prometheus
-LAST DEPLOYED: ...
-NAMESPACE: monitoring
-STATUS: deployed
+
$ cd conf/f3s/prometheus
+$ just install
+kubectl apply -f persistent-volumes.yaml
+persistentvolume/prometheus-data-pv created
+persistentvolume/grafana-data-pv created
+persistentvolumeclaim/grafana-data-pvc created
+helm install prometheus prometheus-community/kube-prometheus-stack \
+    --namespace monitoring -f persistence-values.yaml
+NAME: prometheus
+LAST DEPLOYED: ...
+NAMESPACE: monitoring
+STATUS: deployed
 

 

 The persistence-values.yaml configures Prometheus and Grafana to use the NFS-backed persistent volumes I mentioned earlier, ensuring data survives pod restarts. It also enables scraping of etcd and kube-controller-manager metrics:

@@ -258,12 +253,12 @@ kubeControllerManager:
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
[root@r0 ~]# cat >> /etc/rancher/k3s/config.yaml << 'EOF'
-kube-controller-manager-arg:
-  - bind-address=0.0.0.0
-etcd-expose-metrics: true
-EOF
-[root@r0 ~]# systemctl restart k3s
+
[root@r0 ~]# cat >> /etc/rancher/k3s/config.yaml << 'EOF'
+kube-controller-manager-arg:
+  - bind-address=0.0.0.0
+etcd-expose-metrics: true
+EOF
+[root@r0 ~]# systemctl restart k3s
 

 

 Repeat for r1 and r2. After restarting all nodes, the controller-manager metrics endpoint will be accessible and etcd metrics are available on port 2381. Prometheus can now scrape both.

@@ -274,8 +269,8 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
[root@r0 ~]# curl -s http://127.0.0.1:2381/metrics | grep etcd_server_has_leader
-etcd_server_has_leader 1
+
[root@r0 ~]# curl -s http://127.0.0.1:2381/metrics | grep etcd_server_has_leader
+etcd_server_has_leader 1
 

 

 The full persistence-values.yaml and all other Prometheus configuration files are available on Codeberg:

@@ -296,9 +291,9 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
$ kubectl get svc -n monitoring prometheus-kube-prometheus-prometheus
-NAME                                    TYPE        CLUSTER-IP      PORT(S)
-prometheus-kube-prometheus-prometheus   ClusterIP   10.43.152.163   9090/TCP,8080/TCP
+
$ kubectl get svc -n monitoring prometheus-kube-prometheus-prometheus
+NAME                                    TYPE        CLUSTER-IP      PORT(S)
+prometheus-kube-prometheus-prometheus   ClusterIP   10.43.152.163   9090/TCP,8080/TCP
 

 

 Grafana connects to Prometheus using the internal service URL http://prometheus-kube-prometheus-prometheus.monitoring.svc.cluster.local:9090. The default Grafana credentials are admin/prom-operator, which should be changed immediately after first login.

@@ -323,7 +318,7 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
[root@r0 ~]# mkdir -p /data/nfs/k3svolumes/loki/data
+
[root@r0 ~]# mkdir -p /data/nfs/k3svolumes/loki/data
 

 

 
Deploying Loki and Alloy


@@ -338,24 +333,24 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
$ cd conf/f3s/loki
-$ just install
-helm repo add grafana https://grafana.github.io/helm-charts || true
-helm repo update
-kubectl apply -f persistent-volumes.yaml
-persistentvolume/loki-data-pv created
-persistentvolumeclaim/loki-data-pvc created
-helm install loki grafana/loki --namespace monitoring -f values.yaml
-NAME: loki
-LAST DEPLOYED: ...
-NAMESPACE: monitoring
-STATUS: deployed
-...
-helm install alloy grafana/alloy --namespace monitoring -f alloy-values.yaml
-NAME: alloy
-LAST DEPLOYED: ...
-NAMESPACE: monitoring
-STATUS: deployed
+
$ cd conf/f3s/loki
+$ just install
+helm repo add grafana https://grafana.github.io/helm-charts || true
+helm repo update
+kubectl apply -f persistent-volumes.yaml
+persistentvolume/loki-data-pv created
+persistentvolumeclaim/loki-data-pvc created
+helm install loki grafana/loki --namespace monitoring -f values.yaml
+NAME: loki
+LAST DEPLOYED: ...
+NAMESPACE: monitoring
+STATUS: deployed
+...
+helm install alloy grafana/alloy --namespace monitoring -f alloy-values.yaml
+NAME: alloy
+LAST DEPLOYED: ...
+NAMESPACE: monitoring
+STATUS: deployed
 

 

 Loki runs in single-binary mode with a single replica (loki-0), which is appropriate for a home lab cluster. This means there's only one Loki pod running at any time. If the node hosting Loki fails, Kubernetes will automatically reschedule the pod to another worker node—but there will be a brief downtime (typically under a minute) while this happens. For my home lab use case, this is perfectly acceptable.

@@ -370,44 +365,44 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
discovery.kubernetes "pods" {
-  role = "pod"
-}
+
discovery.kubernetes "pods" {
+  role = "pod"
+}
 
-discovery.relabel "pods" {
-  targets = discovery.kubernetes.pods.targets
+discovery.relabel "pods" {
+  targets = discovery.kubernetes.pods.targets
 
-  rule {
-    source_labels = ["__meta_kubernetes_namespace"]
-    target_label  = "namespace"
-  }
+  rule {
+    source_labels = ["__meta_kubernetes_namespace"]
+    target_label  = "namespace"
+  }
 
-  rule {
-    source_labels = ["__meta_kubernetes_pod_name"]
-    target_label  = "pod"
-  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_name"]
+    target_label  = "pod"
+  }
 
-  rule {
-    source_labels = ["__meta_kubernetes_pod_container_name"]
-    target_label  = "container"
-  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_container_name"]
+    target_label  = "container"
+  }
 
-  rule {
-    source_labels = ["__meta_kubernetes_pod_label_app"]
-    target_label  = "app"
-  }
-}
+  rule {
+    source_labels = ["__meta_kubernetes_pod_label_app"]
+    target_label  = "app"
+  }
+}
 
-loki.source.kubernetes "pods" {
-  targets    = discovery.relabel.pods.output
-  forward_to = [loki.write.default.receiver]
-}
+loki.source.kubernetes "pods" {
+  targets    = discovery.relabel.pods.output
+  forward_to = [loki.write.default.receiver]
+}
 
-loki.write "default" {
-  endpoint {
-    url = "http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push"
-  }
-}
+loki.write "default" {
+  endpoint {
+    url = "http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push"
+  }
+}
 

 

 This configuration automatically labels each log line with the namespace, pod name, container name, and app label, making it easy to filter logs in Grafana.

@@ -420,9 +415,9 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
$ kubectl get svc -n monitoring loki
-NAME   TYPE        CLUSTER-IP    PORT(S)
-loki   ClusterIP   10.43.64.60   3100/TCP,9095/TCP
+
$ kubectl get svc -n monitoring loki
+NAME   TYPE        CLUSTER-IP    PORT(S)
+loki   ClusterIP   10.43.64.60   3100/TCP,9095/TCP
 

 

 To add Loki as a data source in Grafana:

@@ -446,21 +441,21 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
$ kubectl get pods -n monitoring
-NAME                                                     READY   STATUS    RESTARTS   AGE
-alertmanager-prometheus-kube-prometheus-alertmanager-0   2/2     Running   0          42d
-alloy-g5fgj                                              2/2     Running   0          29m
-alloy-nfw8w                                              2/2     Running   0          29m
-alloy-tg9vj                                              2/2     Running   0          29m
-loki-0                                                   2/2     Running   0          25m
-prometheus-grafana-868f9dc7cf-lg2vl                      3/3     Running   0          42d
-prometheus-kube-prometheus-operator-8d7bbc48c-p4sf4      1/1     Running   0          42d
-prometheus-kube-state-metrics-7c5fb9d798-hh2fx           1/1     Running   0          42d
-prometheus-prometheus-kube-prometheus-prometheus-0       2/2     Running   0          42d
-prometheus-prometheus-node-exporter-2nsg9                1/1     Running   0          42d
-prometheus-prometheus-node-exporter-mqr25                1/1     Running   0          42d
-prometheus-prometheus-node-exporter-wp4ds                1/1     Running   0          42d
-tempo-0                                                  1/1     Running   0          1d
+
$ kubectl get pods -n monitoring
+NAME                                                     READY   STATUS    RESTARTS   AGE
+alertmanager-prometheus-kube-prometheus-alertmanager-0   2/2     Running   0          42d
+alloy-g5fgj                                              2/2     Running   0          29m
+alloy-nfw8w                                              2/2     Running   0          29m
+alloy-tg9vj                                              2/2     Running   0          29m
+loki-0                                                   2/2     Running   0          25m
+prometheus-grafana-868f9dc7cf-lg2vl                      3/3     Running   0          42d
+prometheus-kube-prometheus-operator-8d7bbc48c-p4sf4      1/1     Running   0          42d
+prometheus-kube-state-metrics-7c5fb9d798-hh2fx           1/1     Running   0          42d
+prometheus-prometheus-kube-prometheus-prometheus-0       2/2     Running   0          42d
+prometheus-prometheus-node-exporter-2nsg9                1/1     Running   0          42d
+prometheus-prometheus-node-exporter-mqr25                1/1     Running   0          42d
+prometheus-prometheus-node-exporter-wp4ds                1/1     Running   0          42d
+tempo-0                                                  1/1     Running   0          1d
 

 

 Note: Tempo (tempo-0) is deployed later in this post in the "Distributed Tracing with Grafana Tempo" section. It is included in the pod listing here for completeness.

@@ -471,19 +466,19 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
$ kubectl get svc -n monitoring
-NAME                                      TYPE        CLUSTER-IP      PORT(S)
-alertmanager-operated                     ClusterIP   None            9093/TCP,9094/TCP
-alloy                                     ClusterIP   10.43.74.14     12345/TCP
-loki                                      ClusterIP   10.43.64.60     3100/TCP,9095/TCP
-loki-headless                             ClusterIP   None            3100/TCP
-prometheus-grafana                        ClusterIP   10.43.46.82     80/TCP
-prometheus-kube-prometheus-alertmanager   ClusterIP   10.43.208.43    9093/TCP,8080/TCP
-prometheus-kube-prometheus-operator       ClusterIP   10.43.246.121   443/TCP
-prometheus-kube-prometheus-prometheus     ClusterIP   10.43.152.163   9090/TCP,8080/TCP
-prometheus-kube-state-metrics             ClusterIP   10.43.64.26     8080/TCP
-prometheus-prometheus-node-exporter       ClusterIP   10.43.127.242   9100/TCP
-tempo                                     ClusterIP   10.43.91.44     3200/TCP,4317/TCP,4318/TCP
+
$ kubectl get svc -n monitoring
+NAME                                      TYPE        CLUSTER-IP      PORT(S)
+alertmanager-operated                     ClusterIP   None            9093/TCP,9094/TCP
+alloy                                     ClusterIP   10.43.74.14     12345/TCP
+loki                                      ClusterIP   10.43.64.60     3100/TCP,9095/TCP
+loki-headless                             ClusterIP   None            3100/TCP
+prometheus-grafana                        ClusterIP   10.43.46.82     80/TCP
+prometheus-kube-prometheus-alertmanager   ClusterIP   10.43.208.43    9093/TCP,8080/TCP
+prometheus-kube-prometheus-operator       ClusterIP   10.43.246.121   443/TCP
+prometheus-kube-prometheus-prometheus     ClusterIP   10.43.152.163   9090/TCP,8080/TCP
+prometheus-kube-state-metrics             ClusterIP   10.43.64.26     8080/TCP
+prometheus-prometheus-node-exporter       ClusterIP   10.43.127.242   9100/TCP
+tempo                                     ClusterIP   10.43.91.44     3200/TCP,4317/TCP,4318/TCP
 

 

 Let me break down what each pod does:

@@ -560,7 +555,7 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
paul@f0:~ % doas pkg install -y node_exporter
+
paul@f0:~ % doas pkg install -y node_exporter
 

 

 Enable the service to start at boot:

@@ -569,8 +564,8 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
paul@f0:~ % doas sysrc node_exporter_enable=YES
-node_exporter_enable:  -> YES
+
paul@f0:~ % doas sysrc node_exporter_enable=YES
+node_exporter_enable:  -> YES
 

 

 Configure node_exporter to listen on the WireGuard interface. This ensures metrics are only accessible through the secure tunnel, not the public network. Replace the IP with the host's WireGuard address:

@@ -579,8 +574,8 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
paul@f0:~ % doas sysrc node_exporter_args='--web.listen-address=192.168.2.130:9100'
-node_exporter_args:  -> --web.listen-address=192.168.2.130:9100
+
paul@f0:~ % doas sysrc node_exporter_args='--web.listen-address=192.168.2.130:9100'
+node_exporter_args:  -> --web.listen-address=192.168.2.130:9100
 

 

 Start the service:

@@ -589,8 +584,8 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
paul@f0:~ % doas service node_exporter start
-Starting node_exporter.
+
paul@f0:~ % doas service node_exporter start
+Starting node_exporter.
 

 

 Verify it's running:

@@ -599,10 +594,10 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
paul@f0:~ % curl -s http://192.168.2.130:9100/metrics | head -3
-# HELP go_gc_duration_seconds A summary of the wall-time pause...
-# TYPE go_gc_duration_seconds summary
-go_gc_duration_seconds{quantile="0"} 0
+
paul@f0:~ % curl -s http://192.168.2.130:9100/metrics | head -3
+# HELP go_gc_duration_seconds A summary of the wall-time pause...
+# TYPE go_gc_duration_seconds summary
+go_gc_duration_seconds{quantile="0"} 0
 

 

 Repeat for the other FreeBSD hosts (f1, f2) with their respective WireGuard IPs.

@@ -630,9 +625,9 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
$ kubectl create secret generic additional-scrape-configs \
-    --from-file=additional-scrape-configs.yaml \
-    -n monitoring
+
$ kubectl create secret generic additional-scrape-configs \
+    --from-file=additional-scrape-configs.yaml \
+    -n monitoring
 

 

 Update persistence-values.yaml to reference the secret:

@@ -652,7 +647,7 @@ prometheus:
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
$ just upgrade
+
$ just upgrade
 

 

 After a minute or so, the FreeBSD hosts appear in the Prometheus targets and in the Node Exporter dashboards in Grafana.

@@ -1020,10 +1015,10 @@ zfs_pool_free_bytes{pool="zdata"} 3.48809678848e+11
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
blowfish:~ $ doas pkg_add node_exporter
-quirks-7.103 signed on 2025-10-13T22:55:16Z
-The following new rcscripts were installed: /etc/rc.d/node_exporter
-See rcctl(8) for details.
+
blowfish:~ $ doas pkg_add node_exporter
+quirks-7.103 signed on 2025-10-13T22:55:16Z
+The following new rcscripts were installed: /etc/rc.d/node_exporter
+See rcctl(8) for details.
 

 

 Enable the service to start at boot:

@@ -1032,7 +1027,7 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
blowfish:~ $ doas rcctl enable node_exporter
+
blowfish:~ $ doas rcctl enable node_exporter
 

 

 Configure node_exporter to listen on the WireGuard interface. This ensures metrics are only accessible through the secure tunnel, not the public network. Replace the IP with the host's WireGuard address:

@@ -1041,7 +1036,7 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
blowfish:~ $ doas rcctl set node_exporter flags '--web.listen-address=192.168.2.110:9100'
+
blowfish:~ $ doas rcctl set node_exporter flags '--web.listen-address=192.168.2.110:9100'
 

 

 Start the service:

@@ -1050,8 +1045,8 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
blowfish:~ $ doas rcctl start node_exporter
-node_exporter(ok)
+
blowfish:~ $ doas rcctl start node_exporter
+node_exporter(ok)
 

 

 Verify it's running:

@@ -1060,10 +1055,10 @@ http://www.gnu.org/software/src-highlite -->
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
blowfish:~ $ curl -s http://192.168.2.110:9100/metrics | head -3
-# HELP go_gc_duration_seconds A summary of the wall-time pause...
-# TYPE go_gc_duration_seconds summary
-go_gc_duration_seconds{quantile="0"} 0
+
blowfish:~ $ curl -s http://192.168.2.110:9100/metrics | head -3
+# HELP go_gc_duration_seconds A summary of the wall-time pause...
+# TYPE go_gc_duration_seconds summary
+go_gc_duration_seconds{quantile="0"} 0
 

 

 Repeat for the other OpenBSD host (fishfinger) with its respective WireGuard IP (192.168.2.111).

@@ -1425,35 +1420,35 @@ opentelemetry-instrumentation-requests==0.49b0
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
from opentelemetry import trace
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
-from opentelemetry.instrumentation.flask import FlaskInstrumentor
-from opentelemetry.instrumentation.requests import RequestsInstrumentor
-from opentelemetry.sdk.resources import Resource
+
from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.sdk.resources import Resource
 
-# Define service identity
-resource = Resource(attributes={
-    "service.name": "frontend",
-    "service.namespace": "tracing-demo",
-    "service.version": "1.0.0"
-})
+# Define service identity
+resource = Resource(attributes={
+    "service.name": "frontend",
+    "service.namespace": "tracing-demo",
+    "service.version": "1.0.0"
+})
 
-provider = TracerProvider(resource=resource)
+provider = TracerProvider(resource=resource)
 
-# Export to Alloy
-otlp_exporter = OTLPSpanExporter(
-    endpoint="http://alloy.monitoring.svc.cluster.local:4317",
-    insecure=True
-)
+# Export to Alloy
+otlp_exporter = OTLPSpanExporter(
+    endpoint="http://alloy.monitoring.svc.cluster.local:4317",
+    insecure=True
+)
 
-processor = BatchSpanProcessor(otlp_exporter)
-provider.add_span_processor(processor)
-trace.set_tracer_provider(provider)
+processor = BatchSpanProcessor(otlp_exporter)
+provider.add_span_processor(processor)
+trace.set_tracer_provider(provider)
 
-# Auto-instrument Flask and requests
-FlaskInstrumentor().instrument_app(app)
-RequestsInstrumentor().instrument()
+# Auto-instrument Flask and requests
+FlaskInstrumentor().instrument_app(app)
+RequestsInstrumentor().instrument()
 

 

 The auto-instrumentation automatically:

@@ -1630,29 +1625,29 @@ curl -H "Host: tracing-demo.f3s.buetow.org" http://r0/api/process
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
{
-  "middleware_response": {
-    "backend_data": {
-      "data": {
-        "id": 12345,
-        "query_time_ms": 100.0,
-        "timestamp": "2025-12-28T18:35:01.064538",
-        "value": "Sample data from backend service"
-      },
-      "service": "backend"
-    },
-    "middleware_processed": true,
-    "original_data": {
-      "source": "GET request"
-    },
-    "transformation_time_ms": 50
-  },
-  "request_data": {
-    "source": "GET request"
-  },
-  "service": "frontend",
-  "status": "success"
-}
+
{
+  "middleware_response": {
+    "backend_data": {
+      "data": {
+        "id": 12345,
+        "query_time_ms": 100.0,
+        "timestamp": "2025-12-28T18:35:01.064538",
+        "value": "Sample data from backend service"
+      },
+      "service": "backend"
+    },
+    "middleware_processed": true,
+    "original_data": {
+      "source": "GET request"
+    },
+    "transformation_time_ms": 50
+  },
+  "request_data": {
+    "source": "GET request"
+  },
+  "service": "frontend",
+  "status": "success"
+}
 

 

 **2. Find the trace in Tempo via API:**

@@ -1671,12 +1666,12 @@ kubectl exec -n monitoring tempo-0 -- wget -qO- \
 by Lorenzo Bettini
 http://www.lorenzobettini.it
 http://www.gnu.org/software/src-highlite -->
-
{
-  "traceID": "4be1151c0bdcd5625ac7e02b98d95bd5",
-  "rootServiceName": "frontend",
-  "rootTraceName": "GET /api/process",
-  "durationMs": 221
-}
+
{
+  "traceID": "4be1151c0bdcd5625ac7e02b98d95bd5",
+  "rootServiceName": "frontend",
+  "rootTraceName": "GET /api/process",
+  "durationMs": 221
+}
 

 

 **3. Fetch complete trace details:**

@@ -1818,11 +1813,11 @@ kubectl exec -n monitoring <tempo-pod> -- df -h /var/tempo
 

 Back to the main site

 

-    Generated with Gemtexter 3.0.1-develop |
-    served by OpenBSD/relayd(8)+httpd(8) |
-    Site Mirrors
-    

-    Webring: previous | shring | next
+	Generated with Gemtexter 3.0.1-develop |
+	served by OpenBSD/relayd(8)+httpd(8) |
+	Site Mirrors
+	

+	Webring: previous | shring | next
 

 
 
-- 
cgit v1.2.3