#!/bin/bash # Cleanup script: Delete benchmark data from Prometheus # This uses the Prometheus Admin API to selectively remove benchmark metrics # Run from repo root: ./scripts/cleanup-benchmark-data.sh [prometheus_url] set -e PROMETHEUS_URL="${1:-http://localhost:9090}" echo "=== Prometheus Benchmark Data Cleanup ===" echo "" echo "Prometheus URL: $PROMETHEUS_URL" echo "" # Check if port-forward is needed if [[ "$PROMETHEUS_URL" == *"localhost"* ]]; then echo "Note: Make sure you have port-forward running:" echo " kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090" echo "" fi # Metrics to delete METRICS=( "epimetheus_benchmark_cpu_usage" "epimetheus_benchmark_memory_bytes" "epimetheus_benchmark_disk_io_bytes" "epimetheus_benchmark_network_rx_bytes" "epimetheus_benchmark_network_tx_bytes" "epimetheus_benchmark_requests_total" "epimetheus_benchmark_errors_total" "epimetheus_benchmark_response_time_ms" "epimetheus_benchmark_active_connections" "epimetheus_benchmark_queue_depth" ) echo "Step 1: Deleting benchmark metrics..." echo "" SUCCESS_COUNT=0 ERROR_COUNT=0 for METRIC in "${METRICS[@]}"; do echo " Deleting: $METRIC" # Delete series endpoint returns HTTP 204 No Content on success HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST -g "${PROMETHEUS_URL}/api/v1/admin/tsdb/delete_series?match[]=${METRIC}") if [ "$HTTP_CODE" == "204" ] || [ "$HTTP_CODE" == "200" ]; then echo " ✅ Success (HTTP $HTTP_CODE)" SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) else echo " ❌ Error: HTTP $HTTP_CODE" ERROR_COUNT=$((ERROR_COUNT + 1)) fi done echo "" echo "Deletion summary: $SUCCESS_COUNT succeeded, $ERROR_COUNT failed" echo "" if [ $ERROR_COUNT -eq 0 ]; then echo "Step 2: Cleaning up tombstones..." echo "" # Clean tombstones endpoint returns HTTP 204 No Content on success CLEANUP_HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST "${PROMETHEUS_URL}/api/v1/admin/tsdb/clean_tombstones") if [ "$CLEANUP_HTTP_CODE" == "204" ] || [ "$CLEANUP_HTTP_CODE" == "200" ]; then echo " ✅ Tombstones cleaned successfully (HTTP $CLEANUP_HTTP_CODE)" echo "" echo "🎉 Cleanup complete!" echo "" echo "Note: Prometheus may take a few moments to compact the database" echo "and free up disk space." else echo " ❌ Error cleaning tombstones: HTTP $CLEANUP_HTTP_CODE" exit 1 fi else echo "⚠️ Some deletions failed. Skipping tombstone cleanup." echo "Check Prometheus admin API is enabled with:" echo " kubectl get prometheus -n monitoring prometheus-kube-prometheus-prometheus -o yaml | grep -A5 additionalArgs" exit 1 fi echo "" echo "To verify deletion, run:" echo " curl -s '${PROMETHEUS_URL}/api/v1/label/__name__/values' | jq '.data | map(select(startswith(\"epimetheus_benchmark\")))'" echo ""