update pruning script
This commit is contained in:
@@ -7,40 +7,40 @@ source <%= @ps_env_file %>
|
||||
LOG_FILE="<%= @ps_prune_log_file %>"
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') Starting prune run" >> "$LOG_FILE"
|
||||
|
||||
# Find all TSDB blocks
|
||||
TSDB_DIR="/var/lib/prometheus"
|
||||
# -----------------------
|
||||
# Iterate TSDB blocks
|
||||
# -----------------------
|
||||
for block in "$TSDB_DIR"/*/; do
|
||||
[[ -d "$block" ]] || continue
|
||||
meta="$block/meta.json"
|
||||
[[ -f "$meta" ]] || continue
|
||||
|
||||
# Get block min and max timestamps
|
||||
min_time=$(jq -r '.minTime' "$meta")
|
||||
max_time=$(jq -r '.maxTime' "$meta")
|
||||
|
||||
# Skip very recent blocks
|
||||
# Skip last MIN_AGE_HOURS
|
||||
block_age_hours=$(( ($(date +%s) - min_time/1000) / 3600 ))
|
||||
if (( block_age_hours < MIN_AGE_HOURS + GRACE_HOURS )); then
|
||||
echo "Skipping block $block (age ${block_age_hours}h)" >> "$LOG_FILE"
|
||||
log "Keeping block $block (age ${block_age_hours}h)"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Verify block is in PostgreSQL
|
||||
# Check existence in PostgreSQL
|
||||
check_sql="${CHECK_SQL_TEMPLATE//\{min\}/$min_time}"
|
||||
check_sql="${check_sql//\{max\}/$max_time}"
|
||||
|
||||
exists=$(psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -t -c "$check_sql" | xargs)
|
||||
exists=$(psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" -t -c "$check_sql" | xargs || echo "0")
|
||||
|
||||
if [[ "$exists" != "1" ]]; then
|
||||
echo "Block $block not found in PostgreSQL, skipping deletion" >> "$LOG_FILE"
|
||||
log "Block $block not confirmed in PostgreSQL, skipping deletion"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Dry-run or delete
|
||||
# Delete block
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
echo "[DRY-RUN] Would delete block $block" >> "$LOG_FILE"
|
||||
log "[DRY-RUN] Would delete block $block"
|
||||
else
|
||||
echo "Deleting block $block" >> "$LOG_FILE"
|
||||
log "Deleting block $block"
|
||||
rm -rf "$block"
|
||||
fi
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user