diff options
| -rw-r--r-- | README.md | 13 | ||||
| -rw-r--r-- | cmd/silentct-mon/main.go | 6 | ||||
| -rwxr-xr-x | contrib/silentct-check | 104 | ||||
| -rw-r--r-- | docs/metrics.md | 89 | ||||
| -rw-r--r-- | go.mod | 2 | ||||
| -rwxr-xr-x | integration/smoke-test | 4 | ||||
| -rw-r--r-- | internal/manager/manager.go | 31 | ||||
| -rw-r--r-- | internal/metrics/metrics.go | 105 | ||||
| -rw-r--r-- | internal/monitor/backoff.go | 56 | ||||
| -rw-r--r-- | internal/monitor/monitor.go | 3 | ||||
| -rw-r--r-- | pkg/storage/index/index.go | 4 | ||||
| -rw-r--r-- | pkg/storage/index/inmem.go | 16 | ||||
| -rw-r--r-- | pkg/storage/loglist/loglist.go | 4 | ||||
| -rw-r--r-- | pkg/storage/loglist/metadata.go | 26 | ||||
| -rwxr-xr-x | scripts/silentct-check | 96 |
15 files changed, 368 insertions, 191 deletions
@@ -101,6 +101,19 @@ configuration of each trusted system. The monitor will refuse to mark a certificate as legitimate unless the trusted system that requested it had permission to do so. This adds a layer of separation between trusted systems. +The list of logs is configured and kept up-to-date by downloading [Google's +list][] in signed format. To remove a log, specify the `"remove_logs"` list. +Each entry should be a log key on the same format as in the signed list. + + "remove_logs": [ + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE8m/SiQ8/xfiHHqtls9m7FyOMBg4JVZY9CgiixXGz0akvKD6DEL8S0ERmFe9U4ZiA0M4kbT5nmuk3I85Sk4bagA==" + ] + +To add or override a log (mostly useful for debug), specify the `"static_logs"` +list. Each log entry should be on the same format as in the signed list. + +[Google's list]: https://github.com/google/certificate-transparency-community-site/blob/master/docs/google/known-logs.md + ### Start the monitor Start the monitor: diff --git a/cmd/silentct-mon/main.go b/cmd/silentct-mon/main.go index 24a4d03..b8fb912 100644 --- a/cmd/silentct-mon/main.go +++ b/cmd/silentct-mon/main.go @@ -31,9 +31,6 @@ can operate silently, which means there need not be any output unless a certificate is possibly mis-issued. This requires use of the silentct-mac utility on the trusted systems that legitimately request certificates. -The same list of Certificate Transparency logs as Google Chrome is used. This -list can be overridden in the silentct-mon configuration file. - Usage: silentct-mon [Options] -c CONFIGURATION-FILE -d DIRECTORY Options: @@ -46,7 +43,8 @@ Options: -e, --please-exit Toggle to only run until up-to-date (Default: false) -f, --force Override santity checks that may not be fatal (Default: false) -o, --output-file File that all output will be written to (Default: stdout) - -m, --metrics-at Host address to serve /metrics on (Default: disabled) + -m, --metrics-at Host address to serve the Prometheus metrics endpoint + "/metrics" on, e.g., "localhost:12345" (Default: disabled) -p, --pull-interval How often nodes are pulled for certificates (Default: 15m) -v, --verbosity Leveled logging output (default: NOTICE) -w, --num-workers Number of parallel workers to fetch each log with (Default: 1) diff --git a/contrib/silentct-check b/contrib/silentct-check new file mode 100755 index 0000000..712517d --- /dev/null +++ b/contrib/silentct-check @@ -0,0 +1,104 @@ +#!/bin/bash + +# +# A script that outputs warnings from silentct-mon's Prometheus metrics. Mainly +# meant as an example, but may be useful for simple (periodic) one-shot checks. +# + +set -eu + +#------------------------------------------------------------------------------- +# Configuration +#------------------------------------------------------------------------------- +METRICS_AT=${METRICS_AT:-http://localhost:8080/metrics} +ALERT_BACKLOG=${ALERT_BACKLOG:-65536} +ALERT_FRESHNESS=${ALERT_FRESHNESS:-86400} + +#------------------------------------------------------------------------------- +# Helper functions +#------------------------------------------------------------------------------- +notice() { + echo "NOTICE: $*" >&2 +} + +die() { + echo "FATAL: $*" >&2 + exit 1 +} + +to_integer() { + printf "%.f" "$1" +} + +extract_label() { + local line=$1 + local label=$2 + echo "${line#*"$label"=}" | cut -d'"' -f2 +} + +extract_value() { + local line=$1 + echo "${line##* }" +} + +#------------------------------------------------------------------------------- +# Fetch metrics +#------------------------------------------------------------------------------- +metrics_file=$(mktemp) +trap 'rm -f $metrics_file' EXIT + +curl -so "$metrics_file" "$METRICS_AT" || die "failed fetching from $METRICS_AT" + +#------------------------------------------------------------------------------- +# Parse metrics +#------------------------------------------------------------------------------- +declare -A log_index # log ID -> log index +declare -A log_size # log ID -> log size +declare -A log_timestamp # log ID -> log timestamp + +need_restart=0 +num_unexpected_crt=0 + +while IFS= read -r line; do + [[ $line =~ ^# ]] && continue + + case "$line" in + silentct_log_index*) + key=$(extract_label "$line" "log_name") + value=$(to_integer "$(extract_value "$line")") + log_index["$key"]=$value + ;; + silentct_log_size*) + key=$(extract_label "$line" "log_name") + value=$(to_integer "$(extract_value "$line")") + log_size["$key"]=$value + ;; + silentct_log_timestamp*) + key=$(extract_label "$line" "log_name") + value=$(to_integer "$(extract_value "$line")") + log_timestamp["$key"]=$((value / 1000)) + ;; + silentct_need_restart*) + need_restart=$(extract_value "$line") + ;; + silentct_unexpected_certificate*) + num_unexpected_crt=$((num_unexpected_crt + 1)) + ;; + esac +done <"$metrics_file" + +#------------------------------------------------------------------------------- +# Output warnings +#------------------------------------------------------------------------------- +now=$(date +%s) + +for log_name in "${!log_size[@]}"; do + backlog=$((log_size[$log_name] - log_index[$log_name])) + elapsed=$((now - log_timestamp[$log_name])) + + ((backlog < ALERT_BACKLOG)) || notice "$log_name -- backlog is at $backlog" + ((elapsed < ALERT_FRESHNESS)) || notice "$log_name -- latest timestamp at $(date -d @"${log_timestamp[$log_name]}")" +done + +[[ $need_restart == 0 ]] || notice "silentct-mon needs to be restarted" +[[ $num_unexpected_crt == 0 ]] || notice "$num_unexpected_crt unexpected certificate(s)" diff --git a/docs/metrics.md b/docs/metrics.md index 1dea0ef..627776a 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -1,59 +1,62 @@ # Metrics -The `silentct-mon` program emits Prometheus metrics -- enable using the `-m` -option. For a *bash example* of how to create appropriate alerts from these -Prometheus metrics, see [scripts/silentct-check](../scripts/silentct-check). +`silentct-mon` can output Prometheus metrics -- enable using the `-m` option. -## `"silentct_log_size"` +## Examples of useful alerts + + - **The monitor is falling behind on downloading a particular log**, e.g., + `silentct_log_size - silentct_log_index > 65536`. + - **The monitor hasn't seen a fresh timestamp from a particular log**, e.g., + `time() - silentct_log_timestamp > 24*60*60`. + - **The monitor needs restarting**, e.g., `silentct_need_restart != 0` + - **Unexpected certificates have been found**, e.g., + `silentct_unexpected_certificate_count > 0`. + +## `"silentct_error_counter"` ``` -# HELP silentct_log_size The number of entries in the log. -# TYPE silentct_log_size gauge -silentct_log_size{id="TnWjJ1yaEMM4W2zU3z9S6x3w4I4bjWnAsfpksWKaOd8="} 6.07308178e+08 +# HELP silentct_error_counter The number of errors propagated to the main loop. +# TYPE silentct_error_counter counter +silentct_error_counter 0 ``` -`id` is a unique log identifier in base64 (computed as in RFC 6962, §3.2). +Do not use for alerting, this metric is too noisy and currently used for debug. ## `"silentct_log_index"` ``` # HELP silentct_log_index The next log entry to be downloaded. # TYPE silentct_log_index gauge -silentct_log_index{id="TnWjJ1yaEMM4W2zU3z9S6x3w4I4bjWnAsfpksWKaOd8="} 6.07307424e+08 +silentct_log_index{log_id="4e75a3275c9a10c3385b6cd4df3f52eb1df0e08e1b8d69c0b1fa64b1629a39df",log_name="Google 'Argon2025h1'} 7.30980064e+08 ``` -`id` is a unique log identifier in base64 (computed as in RFC 6962, §3.2). +`log_id` is a unique log identifier in hex, computed as in RFC 6962 §3.2. -## `"silentct_log_timestamp"` - -``` -# HELP silentct_log_timestamp The log's UNIX timestamp in ms. -# TYPE silentct_log_timestamp gauge -silentct_log_timestamp{id="TnWjJ1yaEMM4W2zU3z9S6x3w4I4bjWnAsfpksWKaOd8="} 1.735992491111e+12 -``` - -`id` is a unique log identifier in base64 (computed as in RFC 6962, §3.2). +`log_name` is a human-meaningful name of the log. -## `"silentct_certificate_alert"` +## `"silentct_log_size"` ``` -# HELP silentct_certificate_alert The time the certificate without allowlisting was found. -# TYPE silentct_certificate_alert gauge -silentct_certificate_alert{stored_at="/path/to/state/crt_found/<log-hex-id>-<log-index>.json"} 1.735992551e+09 +# HELP silentct_log_size The number of entries in the log. +# TYPE silentct_log_size gauge +silentct_log_size{log_id="4e75a3275c9a10c3385b6cd4df3f52eb1df0e08e1b8d69c0b1fa64b1629a39df",log_name="Google 'Argon2025h1'} 7.31044085e+08 ``` -`stored_at` is where the log entry is stored on the monitor's local file system. -For convenience, the parsed log-entry certificate is also available as `.ascii`. +`log_id` is a unique log identifier in hex, computed as in RFC 6962 §3.2. -## `"silentct_error_counter"` +`log_name` is a human-meaningful name of the log. + +## `"silentct_log_timestamp"` ``` -# HELP silentct_error_counter The number of errors propagated to the main loop. -# TYPE silentct_error_counter counter -silentct_error_counter 0 +# HELP silentct_log_timestamp The log's UNIX timestamp in ms. +# TYPE silentct_log_timestamp gauge +silentct_log_timestamp{log_id="4e75a3275c9a10c3385b6cd4df3f52eb1df0e08e1b8d69c0b1fa64b1629a39df",log_name="Google 'Argon2025h1'} 1.737202578179e+12 ``` -Do not use for alerting, this metric is too noisy and currently used for debug. +`log_id` is a unique log identifier in hex, computed as in RFC 6962 §3.2. + +`log_name` is a human-meaningful name of the log. ## `"silentct_need_restart"` @@ -65,3 +68,29 @@ silentct_need_restart 0 Restarts are normally not needed; but here's a metric until the `silentct-mon` implementation can assure that all corner-cases are handled without restarts. + +## `"silentct_unexpected_certificate_count"` + +``` +# HELP silentct_unexpected_certificate_count Number of certificates without any allowlisting +# TYPE silentct_unexpected_certificate_count gauge +silentct_unexpected_certificate_count{crt_sans="example.org www.example.org",log_id="4e75a3275c9a10c3385b6cd4df3f52eb1df0e08e1b8d69c0b1fa64b1629a39df",log_index="1234",log_name="Google 'Argon2025h1'} 1 +``` + +`crt_sans` are the subject alternative names in the unexpected certificate, +space separated. + +`log_id` is a unique log identifier in hex, computed as in RFC 6962 §3.2. + +`log_index` specifies the log entry that contains the unexpected certificate. + +`log_name` is a human-meaningful name of the log. + +See `STATE_DIRECTORY/crt_found/<log_id>-<log_index>.*` for further details. The +`.json` file contains the downloaded log entry. The `.ascii` file contains the +parsed leaf certificate in a human-readable format to make debugging easier. + +Allowlist an unexpected certificate by ingesting it from a trusted certificate +requester. Alternatively: stop the monitor, manually move the unexpected +certificate from the "alerting" dictionary to the "legitimate" dictionary in +`STATE_DIRECTORY/crt_index.json`, save, and then start the monitor again. @@ -4,6 +4,7 @@ go 1.22.7 require ( github.com/google/certificate-transparency-go v1.3.0 + github.com/google/trillian v1.7.0 github.com/prometheus/client_golang v1.20.5 github.com/transparency-dev/merkle v0.0.2 gitlab.torproject.org/rgdd/ct v0.0.0 @@ -14,7 +15,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/go-logr/logr v1.4.2 // indirect - github.com/google/trillian v1.7.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_model v0.6.1 // indirect diff --git a/integration/smoke-test b/integration/smoke-test index a128592..dacecd0 100755 --- a/integration/smoke-test +++ b/integration/smoke-test @@ -86,7 +86,7 @@ next_domain=$(go run github.com/google/certificate-transparency-go/client/ctclie config "$next_domain" "$remove_keys" >"$dir/config.json" timeout 10s go run ../cmd/silentct-mon -c "$dir/config.json" -d "$dir/state" -C "dev:silentct" -p 1s -v DEBUG 2>&1 | tee "$dir/output.txt" grep -q -E '([1-9][0-9]*|[1-9] matches)' "$dir/output.txt" || die "expected at least one match" -grep -q -F '[NOTICE] certificate mis-issuance?' "$dir/output.txt" || die "expected notice about mis-issued certificate" +grep -q -F '[NOTICE] unexpected certificate:' "$dir/output.txt" || die "expected notice about unexpected certificate" pass "run the monitor and be warned of an unreported certificate" @@ -97,4 +97,4 @@ echo "---" >&2 echo "All smoke tests passed" >&2 echo "For interactive tests:" >&2 echo "go run ../cmd/silentct-mon -c "$dir/config.json" -d "$dir/state" -C "dev:silentct" -p 15s -m localhost:8080 -v DEBUG" >&2 -echo "ALERT_BACKLOG=0 ALERT_FRESHNESS=0 ../scripts/silentct-check" >&2 +echo "ALERT_BACKLOG=0 ALERT_FRESHNESS=0 ../contrib/silentct-check" >&2 diff --git a/internal/manager/manager.go b/internal/manager/manager.go index 90f6507..b839502 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "strings" "time" "gitlab.torproject.org/rgdd/ct/pkg/metadata" @@ -14,6 +15,7 @@ import ( "rgdd.se/silentct/internal/monitor" "rgdd.se/silentct/pkg/policy" "rgdd.se/silentct/pkg/storage" + "rgdd.se/silentct/pkg/storage/loglist" ) type Config struct { @@ -145,7 +147,7 @@ func (mgr *Manager) startupConfig() error { return err } mgr.mconfigCh <- monitor.MonitoredLog{Config: log, State: state} - mgr.Metrics.LogState(state) + mgr.Metrics.LogState(loglist.FormatLogName(log), state) } return nil } @@ -170,7 +172,7 @@ func (mgr *Manager) removeLogs(logs []metadata.Log) { state, _ := mgr.GetMonitorState(log) mgr.Logger.Infof("removing log %s with %d entries in its backlog\n", log.URL, state.TreeSize-state.NextIndex) mgr.mconfigCh <- monitor.MonitoredLog{Config: log} - mgr.Metrics.RemoveLogState(state) + mgr.Metrics.RemoveLogState(loglist.FormatLogName(log), state) } } @@ -187,7 +189,7 @@ func (mgr *Manager) addLogs(ctx context.Context, logs []metadata.Log) { mgr.Logger.Infof("bootstrapping log %s at next index 0\n", log.URL) } mgr.mconfigCh <- monitor.MonitoredLog{Config: log, State: state} - mgr.Metrics.LogState(state) + mgr.Metrics.LogState(loglist.FormatLogName(log), state) } } @@ -209,22 +211,39 @@ func (mgr *Manager) monitorJob(msg monitor.Event) error { if err := mgr.SetMonitorState(msg.State.LogID, msg.State); err != nil { return err } - mgr.Metrics.LogState(msg.State) for _, err := range msg.Errors { mgr.errorJob(err) } + + // no metrics update if the log has just been removed (final event) + name, err := mgr.Storage.LogList.LogName(msg.State.SignedTreeHead.LogID) + if err == nil { + mgr.Metrics.LogState(name, msg.State) + } return nil } func (mgr *Manager) alertJob() error { + // See if there are any new unexpected certificates alerts, err := mgr.Index.TriggerAlerts() if err != nil { return err } for _, alert := range alerts { - mgr.Logger.Noticef("certificate mis-issuance? No allowlisting for %s\n", alert.StoredAt) + mgr.Logger.Noticef("unexpected certificate: no allowlisting for crt_sans=\"%s\", see log_id=\"%x\" log_index=\"%d\"\n", strings.Join(alert.SANs, " "), alert.LogID, alert.LogIndex) + } + + // Update metrics for the current unexpected certificates + alerting := mgr.Storage.Index.Alerting() + var names []string + for _, alert := range alerting { + name, err := mgr.Storage.LogList.LogName(alert.LogID) + if err != nil { + name = "historic log" + } + names = append(names, name) } - mgr.Metrics.CertificateAlert(mgr.Storage.Index.Alerting()) + mgr.Metrics.UnexpectedCertificateCount(names, mgr.Storage.Index.Alerting()) return nil } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index c5ff0d6..aae46cd 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -1,55 +1,51 @@ package metrics import ( + "fmt" + "strings" + "github.com/prometheus/client_golang/prometheus" "rgdd.se/silentct/internal/monitor" "rgdd.se/silentct/pkg/storage/index" ) type Metrics struct { - logSize *prometheus.GaugeVec - logIndex *prometheus.GaugeVec - logTimestamp *prometheus.GaugeVec - certificateAlert *prometheus.GaugeVec - errorCounter prometheus.Counter - needRestart prometheus.Gauge + errorCounter prometheus.Counter + logIndex *prometheus.GaugeVec + logSize *prometheus.GaugeVec + logTimestamp *prometheus.GaugeVec + needRestart prometheus.Gauge + unexpectedCertificateCount *prometheus.GaugeVec } func NewMetrics(registry *prometheus.Registry) *Metrics { m := &Metrics{ - logSize: prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "silentct_log_size", - Help: "The number of entries in the log.", + errorCounter: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "silentct_error_counter", + Help: "The number of errors propagated to the main loop.", }, - []string{"id"}, ), logIndex: prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "silentct_log_index", Help: "The next log entry to be downloaded.", }, - []string{"id"}, + []string{"log_id", "log_name"}, ), - logTimestamp: prometheus.NewGaugeVec( + logSize: prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "silentct_log_timestamp", - Help: "The log's UNIX timestamp in ms.", + Name: "silentct_log_size", + Help: "The number of entries in the log.", }, - []string{"id"}, + []string{"log_id", "log_name"}, ), - certificateAlert: prometheus.NewGaugeVec( + logTimestamp: prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "silentct_certificate_alert", - Help: "The time the certificate without allowlisting was found.", - }, - []string{"stored_at"}, - ), - errorCounter: prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "silentct_error_counter", - Help: "The number of errors propagated to the main loop.", + Name: "silentct_log_timestamp", + Help: "The log's UNIX timestamp in ms.", }, + []string{"log_id", "log_name"}, ), needRestart: prometheus.NewGauge( prometheus.GaugeOpts{ @@ -57,30 +53,55 @@ func NewMetrics(registry *prometheus.Registry) *Metrics { Help: "A non-zero value if the monitor needs restarting.", }, ), + unexpectedCertificateCount: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "silentct_unexpected_certificate_count", + Help: "Number of certificates without any allowlisting", + }, + []string{"log_id", "log_name", "log_index", "crt_sans"}, + ), } - - registry.MustRegister(m.logSize, m.logIndex, m.logTimestamp, m.certificateAlert, m.errorCounter, m.needRestart) + registry.MustRegister( + m.errorCounter, + m.logIndex, + m.logSize, + m.logTimestamp, + m.needRestart, + m.unexpectedCertificateCount, + ) return m } -func (m *Metrics) LogState(state monitor.State) { - id := state.LogID.Base64String() - m.logIndex.WithLabelValues(id).Set(float64(state.NextIndex)) - m.logSize.WithLabelValues(id).Set(float64(state.TreeSize)) - m.logTimestamp.WithLabelValues(id).Set(float64(state.Timestamp)) +func (m *Metrics) LogState(logName string, state monitor.State) { + labels := prometheus.Labels{ + "log_id": fmt.Sprintf("%x", state.LogID[:]), + "log_name": logName, + } + m.logIndex.With(labels).Set(float64(state.NextIndex)) + m.logSize.With(labels).Set(float64(state.TreeSize)) + m.logTimestamp.With(labels).Set(float64(state.Timestamp)) } -func (m *Metrics) RemoveLogState(state monitor.State) { - id := state.LogID.Base64String() - m.logIndex.Delete(prometheus.Labels{"id": id}) - m.logSize.Delete(prometheus.Labels{"id": id}) - m.logTimestamp.Delete(prometheus.Labels{"id": id}) +func (m *Metrics) RemoveLogState(logName string, state monitor.State) { + labels := prometheus.Labels{ + "log_id": fmt.Sprintf("%x", state.LogID[:]), + "log_name": logName, + } + m.logIndex.Delete(labels) + m.logSize.Delete(labels) + m.logTimestamp.Delete(labels) } -func (m *Metrics) CertificateAlert(alerts []index.CertificateInfo) { - m.certificateAlert.Reset() - for _, alert := range alerts { - m.certificateAlert.WithLabelValues(alert.StoredAt).Set(float64(alert.ObservedAt.Unix())) +func (m *Metrics) UnexpectedCertificateCount(logNames []string, alerts []index.CertificateInfo) { + m.unexpectedCertificateCount.Reset() + for i, alert := range alerts { + labels := prometheus.Labels{ + "crt_sans": strings.Join(alert.SANs, " "), + "log_id": fmt.Sprintf("%x", alert.LogID), + "log_name": logNames[i], + "log_index": fmt.Sprintf("%d", alert.LogIndex), + } + m.unexpectedCertificateCount.With(labels).Set(1) } } diff --git a/internal/monitor/backoff.go b/internal/monitor/backoff.go new file mode 100644 index 0000000..63c5f55 --- /dev/null +++ b/internal/monitor/backoff.go @@ -0,0 +1,56 @@ +package monitor + +import ( + "context" + + ct "github.com/google/certificate-transparency-go" + "github.com/google/certificate-transparency-go/client" + "github.com/google/certificate-transparency-go/jsonclient" + "github.com/google/trillian/client/backoff" +) + +// backoffClient wraps client.LogClient so that we always backoff on get-entries +// 4XX and 5XX. Backoff is on by default for get-sth already, and our silentct +// usage is guaranteed to not do any hammering on any of the proof endpoints. +// +// For reference on this issue, see: +// https://github.com/google/certificate-transparency-go/issues/898 +type backoffClient struct { + cli *client.LogClient +} + +func (bc *backoffClient) BaseURI() string { + return bc.cli.BaseURI() +} + +func (bc *backoffClient) GetSTH(ctx context.Context) (*ct.SignedTreeHead, error) { + return bc.cli.GetSTH(ctx) +} + +func (bc *backoffClient) GetSTHConsistency(ctx context.Context, first, second uint64) ([][]byte, error) { + return bc.cli.GetSTHConsistency(ctx, first, second) +} + +func (bc *backoffClient) GetProofByHash(ctx context.Context, hash []byte, treeSize uint64) (*ct.GetProofByHashResponse, error) { + return bc.cli.GetProofByHash(ctx, hash, treeSize) +} + +func (bc *backoffClient) GetRawEntries(ctx context.Context, start, end int64) (*ct.GetEntriesResponse, error) { + rsp, err := bc.cli.GetRawEntries(ctx, start, end) + if err != nil { + jcErr, ok := err.(jsonclient.RspError) + if !ok { + return rsp, err + } + if jcErr.StatusCode < 400 || jcErr.StatusCode >= 600 { + return rsp, err + } + // This ensures we never start hammering when the status code is 4XX or + // 5XX. Probably not the right thing to do in all cases, but since the + // download library we're using starts hammering if the log suddenly + // serves something unexpected this seems like a good safety precaution. + // Users of the silentct monitor eventually notice they get no entries. + return rsp, backoff.RetriableErrorf("get-entries: %v", err) + } + return rsp, err +} diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index 1f068b2..2575977 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -173,7 +173,8 @@ func (mon *Monitor) newTailRFC6962(log MonitoredLog) (tail, error) { return tail{}, err } - return tail{cfg: mon.cfg, scanner: cli, checker: cli, matcher: mon.matcher}, nil + bc := &backoffClient{cli: cli} + return tail{cfg: mon.cfg, scanner: bc, checker: bc, matcher: mon.matcher}, nil } func (mon *Monitor) newTailTile(cfg MonitoredLog) (tail, error) { diff --git a/pkg/storage/index/index.go b/pkg/storage/index/index.go index 95eb04a..c85a9e9 100644 --- a/pkg/storage/index/index.go +++ b/pkg/storage/index/index.go @@ -56,7 +56,7 @@ func (ix *Index) AddChain(node string, pem []byte) error { var crtID CertificateID crtID.Set(chain[0]) path := fmt.Sprintf("%s/%s-%s.pem", ix.cfg.TrustDirectory, node, crtID) - if !ix.mem.addChain(crtID, path) { + if !ix.mem.addChain(path, crtID, chain[0].DNSNames) { return nil // duplicate } @@ -76,7 +76,7 @@ func (ix *Index) AddEntries(logID [sha256.Size]byte, entries []monitor.LogEntry) var crtID CertificateID crtID.Set(crt) path := fmt.Sprintf("%s/%x-%d.json", ix.cfg.MatchDirectory, logID[:], entry.LeafIndex) - if !ix.mem.addEntry(crtID, path) { + if !ix.mem.addEntry(path, crtID, crt.DNSNames, logID, entry.LeafIndex) { return nil // duplicate } if err := ioutil.CommitJSON(path, entry); err != nil { diff --git a/pkg/storage/index/inmem.go b/pkg/storage/index/inmem.go index ba48bc1..6184cad 100644 --- a/pkg/storage/index/inmem.go +++ b/pkg/storage/index/inmem.go @@ -16,8 +16,12 @@ func (crtID *CertificateID) Set(crt x509.Certificate) { } type CertificateInfo struct { - ObservedAt time.Time `json:"observed_at"` - StoredAt string `json:"stored_at"` + ObservedAt time.Time `json:"observed_at"` + StoredAt string `json:"stored_at"` + SerialNumber CertificateID `json:"serial_number"` + SANs []string `json:"crt_sans"` + LogID [32]byte `json:"log_id,omitempty"` + LogIndex uint64 `json:"log_index,omitempty"` } // index is an in-memory index of certificates @@ -65,12 +69,12 @@ func (ix *index) triggerAlerts(delay time.Duration) []CertificateInfo { return alerts } -func (ix *index) addChain(crtID CertificateID, path string) bool { +func (ix *index) addChain(path string, crtID CertificateID, sans []string) bool { if _, ok := ix.Legitimate[crtID]; ok { return false // we already marked this certificate as "good" } - entry := CertificateInfo{ObservedAt: time.Now(), StoredAt: path} + entry := CertificateInfo{ObservedAt: time.Now(), StoredAt: path, SerialNumber: crtID, SANs: sans} crtInfos := []CertificateInfo{entry} if v, ok := ix.Alerting[crtID]; ok { crtInfos = append(crtInfos, v...) @@ -84,8 +88,8 @@ func (ix *index) addChain(crtID CertificateID, path string) bool { return true // index updated such that this certificate is marked as "good" } -func (ix *index) addEntry(crtID CertificateID, path string) bool { - crtInfo := CertificateInfo{ObservedAt: time.Now(), StoredAt: path} +func (ix *index) addEntry(path string, crtID CertificateID, sans []string, logID [32]byte, logIndex uint64) bool { + crtInfo := CertificateInfo{ObservedAt: time.Now(), StoredAt: path, SerialNumber: crtID, SANs: sans, LogID: logID, LogIndex: logIndex} if _, ok := ix.Legitimate[crtID]; ok { return add(ix.Legitimate, crtID, crtInfo) } else if _, ok := ix.Alerting[crtID]; ok { diff --git a/pkg/storage/loglist/loglist.go b/pkg/storage/loglist/loglist.go index a37cb32..f282113 100644 --- a/pkg/storage/loglist/loglist.go +++ b/pkg/storage/loglist/loglist.go @@ -72,6 +72,10 @@ func New(cfg Config) (LogList, error) { return ll, nil } +func (ll *LogList) LogName(logID [32]byte) (string, error) { + return metadataLogName(ll.md, logID) +} + func (ll *LogList) IsRecent() bool { return time.Now().Before(ll.md.CreatedAt.Add(ll.cfg.MetadataIsRecent)) } diff --git a/pkg/storage/loglist/metadata.go b/pkg/storage/loglist/metadata.go index adacf81..96d035c 100644 --- a/pkg/storage/loglist/metadata.go +++ b/pkg/storage/loglist/metadata.go @@ -1,6 +1,11 @@ package loglist -import "gitlab.torproject.org/rgdd/ct/pkg/metadata" +import ( + "fmt" + "strings" + + "gitlab.torproject.org/rgdd/ct/pkg/metadata" +) // FIXME: helpers that should probably be in the upstream package @@ -13,6 +18,25 @@ func metadataFindLog(md metadata.Metadata, target metadata.Log) bool { return false } +func metadataLogName(md metadata.Metadata, targetID [32]byte) (string, error) { + for _, operator := range md.Operators { + for _, log := range operator.Logs { + id, _ := log.Key.ID() + if id == targetID { + return FormatLogName(log), nil + } + } + } + return "", fmt.Errorf("no match for log ID: %x", targetID[:]) +} + +func FormatLogName(log metadata.Log) string { + if log.Description != nil { + return *log.Description + } + return strings.TrimSuffix("https://", string(log.URL)) +} + func findLog(logs []metadata.Log, target metadata.Log) bool { targetID, _ := target.Key.ID() for _, log := range logs { diff --git a/scripts/silentct-check b/scripts/silentct-check deleted file mode 100755 index 2c64d67..0000000 --- a/scripts/silentct-check +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash - -# -# A script that generates alerts based on the the silentct-mon prometheus -# metrics. Mainly meant as an example on how to define relevant alerts. -# - -set -eu - -function notice() { - echo "NOTICE: $*" >&2 -} - -function die() { - echo "FATAL: $*" >&2 - exit 1 -} - -#----------------------------------------------------------------------------------------- -# Options -#----------------------------------------------------------------------------------------- -METRICS_AT=${METRICS_AT:-http://localhost:8080/metrics} -ALERT_BACKLOG=${ALERT_BACKLOG:-65536} -ALERT_FRESHNESS=${ALERT_FRESHNESS:-86400} - -#----------------------------------------------------------------------------------------- -# Download the current prometheus metrics -#----------------------------------------------------------------------------------------- -metrics_file=$(mktemp) -trap "rm -f $metrics_file" EXIT -curl -so "$metrics_file" "$METRICS_AT" || die "failed retrieving metrics from $METRICS_AT" - -#----------------------------------------------------------------------------------------- -# Parse metrics -#----------------------------------------------------------------------------------------- -declare -A log_index -declare -A log_size -declare -A log_timestamp -declare -A certificate_alert -while IFS= read -r line; do - if [[ $line =~ ^# ]]; then - continue # skip comments - fi - - if [[ $line =~ ^silentct_log_index ]]; then - id=$(echo "$line" | grep -oP '(?<=id=")[^"]+') - value=$(echo "$line" | awk '{print $NF}') - log_index["$id"]=$value - fi - - if [[ $line =~ ^silentct_log_size ]]; then - id=$(echo "$line" | grep -oP '(?<=id=")[^"]+') - value=$(echo "$line" | awk '{print $NF}') - log_size["$id"]=$value - fi - - if [[ $line =~ ^silentct_log_timestamp ]]; then - id=$(echo "$line" | grep -oP '(?<=id=")[^"]+') - value=$(echo "$line" | awk '{print $NF}') - log_timestamp["$id"]=$value - fi - - if [[ $line =~ ^silentct_certificate_alert ]]; then - stored_at=$(echo "$line" | grep -oP '(?<=stored_at=")[^"]+') - observed_at=$(echo "$line" | awk '{print $NF}') - certificate_alert["$stored_at"]=$observed_at - fi -done <"$metrics_file" - -line=$(grep "^silentct_need_restart" "$metrics_file") -need_restart=$(echo $line | awk '{print $NF}') - -#----------------------------------------------------------------------------------------- -# Output alerts -#----------------------------------------------------------------------------------------- -now=$(date +%s) -for id in "${!log_size[@]}"; do - backlog=$(awk "BEGIN {print ${log_size[$id]} - ${log_index[$id]}}") - if awk "BEGIN {exit !($backlog - $ALERT_BACKLOG >= 0)}"; then - notice "log $id -- backlog is at $backlog" - fi - - unix_timestamp=$(awk "BEGIN {printf \"%.0f\", ${log_timestamp[$id]} / 1000}") - if (( now - unix_timestamp >= ALERT_FRESHNESS )); then - notice "log $id -- latest timestamp at $(date -d @$unix_timestamp)" - fi -done - -for stored_at in "${!certificate_alert[@]}"; do - observed_at=$(awk "BEGIN {printf \"%.0f\", ${certificate_alert[$stored_at]}}") - notice "(mis)-issued certificate? Observed at $(date -d @$observed_at) -- see $stored_at" -done - -if [[ $need_restart != 0 ]]; then - notice "silentct-mon needs to be restarted" -fi |