prometheus: Add basic metrics for alerting

- Detect if we're falling behind while downloading
- Detect if there are any found certificates alerting

2 files changed, 22 insertions, 0 deletions

diff --git a/cmd/silentct-mon/main.go b/cmd/silentct-mon/main.go
index e2ecdb7..ce0a548 100644
--- a/cmd/silentct-mon/main.go
+++ b/cmd/silentct-mon/main.go
@@ -6,6 +6,7 @@ import (
 	"flag"
 	"fmt"
 	"log"
+	"net/http"
 	"os"
 	"os/signal"
 	"strings"
@@ -13,11 +14,14 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"rgdd.se/silentct/internal/feedback"
 	"rgdd.se/silentct/internal/flagopt"
 	"rgdd.se/silentct/internal/ioutil"
 	"rgdd.se/silentct/internal/logger"
 	"rgdd.se/silentct/internal/manager"
+	"rgdd.se/silentct/internal/metrics"
 	"rgdd.se/silentct/internal/monitor"
 	"rgdd.se/silentct/pkg/policy"
 )
@@ -43,6 +47,7 @@ Options:
   -e, --please-exit    Toggle to only run until up-to-date (Default: false)
   -f, --force          Override santity checks that may not be fatal (Default: false)
   -o, --output-file    File that all output will be written to (Default: stdout)
+  -m, --metrics-at     Host address to serve /metrics on (Default: disabled)
   -p, --pull-interval  How often nodes are pulled for certificates (Default: 15m)
   -v, --verbosity      Leveled logging output (default: NOTICE)
   -w, --num-workers    Number of parallel workers to fetch each log with (Default: 1)
@@ -57,6 +62,7 @@ type config struct {
 	directory    string
 	pleaseExit   bool
 	force        bool
+	metricsAt    string
 	outputFile   string
 	pullInterval time.Duration
 	numWorkers   uint
@@ -75,6 +81,7 @@ func configure(cmd string, args []string) (cfg config, err error) {
 	flagopt.StringOpt(fs, &cfg.directory, "directory", "d", "")
 	flagopt.BoolOpt(fs, &cfg.pleaseExit, "please-exit", "e", false)
 	flagopt.BoolOpt(fs, &cfg.force, "force", "f", false)
+	flagopt.StringOpt(fs, &cfg.metricsAt, "metrics-at", "m", "")
 	flagopt.StringOpt(fs, &cfg.outputFile, "output-file", "o", "")
 	flagopt.DurationOpt(fs, &cfg.pullInterval, "pull-interval", "p", 15*time.Minute)
 	flagopt.StringOpt(fs, &cfg.verbosity, "verbosity", "v", logger.LevelNotice.String())
@@ -144,10 +151,12 @@ func main() {
 	errorCh := make(chan error)
 	defer close(errorCh)
 
+	registry := prometheus.NewRegistry()
 	mgr, err := manager.New(manager.Config{
 		Policy:     cfg.policy,
 		Bootstrap:  cfg.bootstrap,
 		Directory:  cfg.directory,
+		Metrics:    metrics.NewMetrics(registry),
 		Logger:     cfg.log,
 		AlertDelay: cfg.pullInterval * 3 / 2,
 	}, feventCh, meventCh, mconfigCh, errorCh)
@@ -203,6 +212,19 @@ func main() {
 		fb.RunForever(ctx)
 	}()
 
+	if cfg.metricsAt != "" {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			defer cancel()
+
+			http.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
+			if err := http.ListenAndServe(cfg.metricsAt, nil); err != nil {
+				cfg.log.Fatalf("metrics: %v\n", err)
+			}
+		}()
+	}
+
 	os.Exit(func() int {
 		defer wg.Wait()
 		defer cancel()
diff --git a/cmd/silentct-mon/silentct-mon b/cmd/silentct-mon/silentct-mon
new file mode 100755
index 0000000..1965f48
--- /dev/null
+++ b/cmd/silentct-mon/silentct-mon