diff options
| -rwxr-xr-x | scripts/silentct-check | 34 |
1 files changed, 11 insertions, 23 deletions
diff --git a/scripts/silentct-check b/scripts/silentct-check index a6a79a3..2c64d67 100755 --- a/scripts/silentct-check +++ b/scripts/silentct-check @@ -1,14 +1,14 @@ #!/bin/bash # -# A script that emits warnings based on the the silentct-mon prometheus metrics. -# Mainly meant as an example for those that configure alerts using prometheus. +# A script that generates alerts based on the the silentct-mon prometheus +# metrics. Mainly meant as an example on how to define relevant alerts. # set -eu -function warn() { - echo "WARNING: $*" >&2 +function notice() { + echo "NOTICE: $*" >&2 } function die() { @@ -31,11 +31,12 @@ trap "rm -f $metrics_file" EXIT curl -so "$metrics_file" "$METRICS_AT" || die "failed retrieving metrics from $METRICS_AT" #----------------------------------------------------------------------------------------- -# Parse per-log metrics +# Parse metrics #----------------------------------------------------------------------------------------- declare -A log_index declare -A log_size declare -A log_timestamp +declare -A certificate_alert while IFS= read -r line; do if [[ $line =~ ^# ]]; then continue # skip comments @@ -58,16 +59,6 @@ while IFS= read -r line; do value=$(echo "$line" | awk '{print $NF}') log_timestamp["$id"]=$value fi -done <"$metrics_file" - -#----------------------------------------------------------------------------------------- -# Parse certificate-alert metrics -#----------------------------------------------------------------------------------------- -declare -A certificate_alert -while IFS= read -r line; do - if [[ $line =~ ^# ]]; then - continue # skip comments - fi if [[ $line =~ ^silentct_certificate_alert ]]; then stored_at=$(echo "$line" | grep -oP '(?<=stored_at=")[^"]+') @@ -76,33 +67,30 @@ while IFS= read -r line; do fi done <"$metrics_file" -#----------------------------------------------------------------------------------------- -# Parse restart metric -#----------------------------------------------------------------------------------------- line=$(grep "^silentct_need_restart" "$metrics_file") need_restart=$(echo $line | awk '{print $NF}') #----------------------------------------------------------------------------------------- -# Emit warnings +# Output alerts #----------------------------------------------------------------------------------------- now=$(date +%s) for id in "${!log_size[@]}"; do backlog=$(awk "BEGIN {print ${log_size[$id]} - ${log_index[$id]}}") if awk "BEGIN {exit !($backlog - $ALERT_BACKLOG >= 0)}"; then - warn "log $id -- backlog is at $backlog" + notice "log $id -- backlog is at $backlog" fi unix_timestamp=$(awk "BEGIN {printf \"%.0f\", ${log_timestamp[$id]} / 1000}") if (( now - unix_timestamp >= ALERT_FRESHNESS )); then - warn "log $id -- latest timestamp at $(date -d @$unix_timestamp)" + notice "log $id -- latest timestamp at $(date -d @$unix_timestamp)" fi done for stored_at in "${!certificate_alert[@]}"; do observed_at=$(awk "BEGIN {printf \"%.0f\", ${certificate_alert[$stored_at]}}") - warn "(mis)-issued certificate? Observed at $(date -d @$observed_at) -- see $stored_at" + notice "(mis)-issued certificate? Observed at $(date -d @$observed_at) -- see $stored_at" done if [[ $need_restart != 0 ]]; then - warn "silentct-mon needs to be restarted" + notice "silentct-mon needs to be restarted" fi |