diff options
31 files changed, 1006 insertions, 279 deletions
@@ -1 +1,2 @@ build +integration/testonly diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index acea3ec..149a462 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: golang:1.19 +image: golang:1.23 stages: - test @@ -16,3 +16,4 @@ is governed by different license requirements than listed in the LICENSE file. Authors, in chronological order of initial contribution: Rasmus Dahlberg + Filippo Valsorda @@ -0,0 +1,4 @@ +NEWS beta.1-WIP + + Initial release of the silentct software. See the RELEASES file for + information on what can be expected of the upcoming beta releases. @@ -2,83 +2,84 @@ An implementation of a silent Certificate Transparency monitor. -**Status:** drafty prototype, please do not use for anything serious. +## Status + +Drafty prototype, please do not use for anything too serious yet. See +[RELEASES](./RELEASES) for information on what can be expected right now. ## How it works A monitor downloads all certificates from Certificate Transparency logs. This -gives a concise view of which certificates have been issued for what domains. +provides a concise view of which certificates have been issued for what domains. -The monitor is configured to pull legitimately issued certificates from trusted -nodes. Each node packages its legitimate certificates as a single submission. -This submission is then made available for download on a URL via HTTP GET. +The monitor is configured to pull certificates from trusted systems that +legitimately request certificates. The legitimately issued certificates are +typically made available on HTTP(S) URLs that are polled periodically. -To convince the monitor that a submission was generated by a trusted node, a -shared secret is established between the node and the monitor. The node creates -a message authentication code using the shared secret. The monitor verifies it. +To convince the monitor that the file of legitimately issued certificates is +authentic, it is integrity protected using a message authentication code. So, +the monitor and its trusted systems need to be configured with shared secrets. -What makes this setup _silent_ is the fact that the monitor can compute the -difference between any discovered and legitimately issued certificates. If a -certificate is found that no node submitted, only then is an alert printed. +What makes this setup "silent" is that the monitor can compute the difference +between any downloaded and legitimately issued certificates. If a certificate +is found that no trusted system made available, only then an alert is emitted. -## Quick start +See the [silentct design](./docs/design.md) for a lengthier introduction. -### Setup a node +## Quickstart -You will need the `silentct-mac` tool to create submissions that the monitor can -pull. Install: +### Setup a trusted system - $ go install rgdd.se/silentct/cmd/silentct-mac@latest +Install the `silentct-mac` tool. -Locate the node's certificates that are still valid (i.e., not expired) and -prepare a submission for them: + $ go install rgdd.se/silentct/cmd/silentct-mac@latest - $ silentct-mac -n NAME -s SECRET /path/to/chain-1.pem /path/to/chain-2.pem ... +Mark all certificates that have yet to expire as legitimately issued. The below +specifies one certificate chain, but it is possible to list multiple ones. -`NAME` is an arbitrary name of the node. + $ silentct-mac -n example.org -s sikritpassword /etc/letsencrypt/live/example.org/fullchain.pem -`SECRET` is a secret that will only be shared with the monitor. +`-n` sets an arbitrary name of the trusted system. -The output includes the node name, the computed message authentication code, and -the list of certificate chains that was specified. Use the `-o` option if you -prefer to save the output directly to a file rather than getting it on stdout. +`-s` sets a secret that will be shared with the monitor. -Make the generated submission file available on a URL. Typically each node -already serves web content, in which case you can copy it into some directory. +The output includes the name, a message authentication code, and the list of +certificate chains that was specified. Record the output as a file the monitor +can pull, e.g., by saving it in a web root or transferring it to one. Ensure +that this file gets updated each time a new certificate is legitimately issued. +Below is an example that keeps the file up-to-date with `crontab` and `certbot`. -Finally, ensure that anytime the node requests a new certificate to be issued, -then a new submission is generated that replaces or extends the previous one. -For example, if `certbot` is run by `cron`, then that is a good place to hook. + # crontab -l + 35 3 * * * certbot renew --post-hook "silentct-mac -n example.org -s sikritpassword -o /var/www/example.org/silentct/allowlist /etc/letsencrypt/live/example.org/fullchain.pem" -Repeat this setup if there are multiple nodes. +**Note:** the `--post-hook` option can be specified per site in the +`[renewalparams]` section of `/etc/letsencrypt/renewal/example.org.conf`. ### Setup the monitor -Install on the system that will run the monitor: +Install the `silentct-mon` tool: $ go install rgdd.se/silentct/cmd/silentct-mon@latest -Create a monitor policy file in JSON format. Below is an example that looks for -all certificates related to `example.org`, expect for certificates that are -associated with `test.example.org`. You need at least one wildcard to match on. +Create a configuration file. - $ cat policy.json + $ cat config.json { "monitor": [ { "bootstrap_at": "2024-01-01T00:00:00Z", - "wildcard": "example.org", + "suffix": "example.org", "excludes": [ "test" ] } ], - "nodes": [ + "certificate_requesters": [ { - "name": "NODE_NAME", - "secret": "NODE_SECRET", - "url": "SUBMISSION_URL", - "issues": [ + "name": "example.org", + "secret": "sikritpassword", + "location": "https://www.example.org/silentct/allowlist", + "requests": [ "example.org", "www.example.org" ] @@ -86,32 +87,56 @@ associated with `test.example.org`. You need at least one wildcard to match on. ] } -The `excludes` keyword is optional. The `bootstrap_at` keyword is required, and -should be set to the current time of adding the wildcard for monitoring. Any -certificate that expired before the specified bootstrap time will be ignored. +`bootstrap_at` is the time the monitor started looking for certificates that +match `suffix`. The monitor considers a certificate to match iff (i) it expired +after the bootstrap time, and (ii) at least one subject alternative name ends +with the specified suffix without a longer match being available when taking the +optional `excludes` list of subdomains into account. For example, the above +configuration matches `www.example.org` but not `foo.test.example.org`. + +Each entry in the `"certificate-requesters"` list corresponds to a trusted +system and the domains it requests certificates for. Set `name`, `secret`, +`location` (filename or URL to pull from), and `requests` to match the +configuration of each trusted system. The monitor will refuse to mark a +certificate as legitimate unless the trusted system that requested it had +permission to do so. This adds a layer of separation between trusted systems. + +The list of logs is configured and kept up-to-date by downloading [Google's +list][] in signed format. To remove a log, specify the `"remove_logs"` list. +Each entry should be a log key on the same format as in the signed list. + + "remove_logs": [ + "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE8m/SiQ8/xfiHHqtls9m7FyOMBg4JVZY9CgiixXGz0akvKD6DEL8S0ERmFe9U4ZiA0M4kbT5nmuk3I85Sk4bagA==" + ] + +To add or override a log (mostly useful for debug), specify the `"static_logs"` +list. Each log entry should be on the same format as in the signed list. + +[Google's list]: https://github.com/google/certificate-transparency-community-site/blob/master/docs/google/known-logs.md + +### Start the monitor -Populate the list of nodes based on the names, secrets, and URLs selected during -your setup. Also add the domains each node is allowed to put into certificates. +Start the monitor: -Bootstrap the monitor in a non-existent directory: + $ silentct-mon -c config.json -d ~/.local/lib/silentct - $ silentct-mon --bootstrap -f policy.json -d /path/to/directory -v INFO - ... +Use the `--bootstrap` flag when running the monitor for the first time. -Leave the monitor running: +Noteworthy events will be printed on stdout using the NOTICE level. If you +prefer to get the monitor's output in a file, use the `-o` option. - $ silentct-mon -f policy.json -d /path/to/directory +### Stop the monitor -Any noteworthy events (like a potentially mis-issued certificate that no node -submitted) will be printed on stdout. If you prefer to get the monitor's output -in a file, use the `-o` option. Please note that it is your job to watch the -output, and/or to hook it up to a notification system like email or similar. +Stop the monitor gracefully by sending the SIGINT or SIGTERM signals. Nothing +bad will happen on an ungraceful exit (just redundant work on next startup). -### Further documentation +## Issue tracker - - A lengthier [introduction](./docs/introduction.md) to the overall design + - https://git.glasklar.is/rgdd/silentct/-/issues + - Email rgdd-silentct-issues (at) incoming (dot) glasklar (dot) is ## Contact - Room `#certificate-transparency` at OFTC.net - Room `#certificate-transparency` at matrix.org + - Email rgdd (at) glasklarteknik (dot) se diff --git a/RELEASES b/RELEASES new file mode 100644 index 0000000..0c2e0e1 --- /dev/null +++ b/RELEASES @@ -0,0 +1,28 @@ +---Overview + +The silentct software is not stable and well-tested yet. Please read the NEWS +file for information on how to upgrade from one beta version to the next. + +The git-tag convention for versioning is + + beta.NUM + +where NUM is a monotonic counter. + +By the terms of the LICENSE you are free to use this code "as is" in almost any +way you like. But for now, beta-use is only supported via the programs in cmd/. +This means the NEWS file is only targeted towards users of the cmd/ programs. + +See + + https://git.glasklar.is/rgdd/silentct/-/issues + +for an outlook of what we might be working on next. + +---Release cheat-sheet + + * Makefile works and produces reasonable man pages + * Quickstart instructions in the README file works + * NEWS file describes what changed from a cmd/ perspective + * Bump version in README + * Push next beta.NUM tag diff --git a/cmd/silentct-mon/main.go b/cmd/silentct-mon/main.go index e2ecdb7..b8fb912 100644 --- a/cmd/silentct-mon/main.go +++ b/cmd/silentct-mon/main.go @@ -5,7 +5,7 @@ import ( "errors" "flag" "fmt" - "log" + "net/http" "os" "os/signal" "strings" @@ -13,11 +13,14 @@ import ( "syscall" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "rgdd.se/silentct/internal/feedback" "rgdd.se/silentct/internal/flagopt" "rgdd.se/silentct/internal/ioutil" "rgdd.se/silentct/internal/logger" "rgdd.se/silentct/internal/manager" + "rgdd.se/silentct/internal/metrics" "rgdd.se/silentct/internal/monitor" "rgdd.se/silentct/pkg/policy" ) @@ -28,9 +31,6 @@ can operate silently, which means there need not be any output unless a certificate is possibly mis-issued. This requires use of the silentct-mac utility on the trusted systems that legitimately request certificates. -The same list of Certificate Transparency logs as Google Chrome is used. This -list can be overridden in the silentct-mon configuration file. - Usage: silentct-mon [Options] -c CONFIGURATION-FILE -d DIRECTORY Options: @@ -43,6 +43,8 @@ Options: -e, --please-exit Toggle to only run until up-to-date (Default: false) -f, --force Override santity checks that may not be fatal (Default: false) -o, --output-file File that all output will be written to (Default: stdout) + -m, --metrics-at Host address to serve the Prometheus metrics endpoint + "/metrics" on, e.g., "localhost:12345" (Default: disabled) -p, --pull-interval How often nodes are pulled for certificates (Default: 15m) -v, --verbosity Leveled logging output (default: NOTICE) -w, --num-workers Number of parallel workers to fetch each log with (Default: 1) @@ -57,6 +59,7 @@ type config struct { directory string pleaseExit bool force bool + metricsAt string outputFile string pullInterval time.Duration numWorkers uint @@ -75,6 +78,7 @@ func configure(cmd string, args []string) (cfg config, err error) { flagopt.StringOpt(fs, &cfg.directory, "directory", "d", "") flagopt.BoolOpt(fs, &cfg.pleaseExit, "please-exit", "e", false) flagopt.BoolOpt(fs, &cfg.force, "force", "f", false) + flagopt.StringOpt(fs, &cfg.metricsAt, "metrics-at", "m", "") flagopt.StringOpt(fs, &cfg.outputFile, "output-file", "o", "") flagopt.DurationOpt(fs, &cfg.pullInterval, "pull-interval", "p", 15*time.Minute) flagopt.StringOpt(fs, &cfg.verbosity, "verbosity", "v", logger.LevelNotice.String()) @@ -144,10 +148,12 @@ func main() { errorCh := make(chan error) defer close(errorCh) + registry := prometheus.NewRegistry() mgr, err := manager.New(manager.Config{ Policy: cfg.policy, Bootstrap: cfg.bootstrap, Directory: cfg.directory, + Metrics: metrics.NewMetrics(registry), Logger: cfg.log, AlertDelay: cfg.pullInterval * 3 / 2, }, feventCh, meventCh, mconfigCh, errorCh) @@ -203,11 +209,24 @@ func main() { fb.RunForever(ctx) }() + if cfg.metricsAt != "" { + wg.Add(1) + go func() { + defer wg.Done() + defer cancel() + + http.Handle("/metrics", promhttp.HandlerFor(registry, promhttp.HandlerOpts{})) + if err := http.ListenAndServe(cfg.metricsAt, nil); err != nil { + cfg.log.Fatalf("metrics: %v\n", err) + } + }() + } + os.Exit(func() int { defer wg.Wait() defer cancel() if err := mgr.Run(ctx); err != nil { - log.Fatalf("manager: %v\n", err) + cfg.log.Fatalf("manager: %v\n", err) return 1 } return 0 diff --git a/cmd/silentct-mon/silentct-mon b/cmd/silentct-mon/silentct-mon Binary files differnew file mode 100755 index 0000000..1965f48 --- /dev/null +++ b/cmd/silentct-mon/silentct-mon diff --git a/contrib/silentct-check b/contrib/silentct-check new file mode 100755 index 0000000..712517d --- /dev/null +++ b/contrib/silentct-check @@ -0,0 +1,104 @@ +#!/bin/bash + +# +# A script that outputs warnings from silentct-mon's Prometheus metrics. Mainly +# meant as an example, but may be useful for simple (periodic) one-shot checks. +# + +set -eu + +#------------------------------------------------------------------------------- +# Configuration +#------------------------------------------------------------------------------- +METRICS_AT=${METRICS_AT:-http://localhost:8080/metrics} +ALERT_BACKLOG=${ALERT_BACKLOG:-65536} +ALERT_FRESHNESS=${ALERT_FRESHNESS:-86400} + +#------------------------------------------------------------------------------- +# Helper functions +#------------------------------------------------------------------------------- +notice() { + echo "NOTICE: $*" >&2 +} + +die() { + echo "FATAL: $*" >&2 + exit 1 +} + +to_integer() { + printf "%.f" "$1" +} + +extract_label() { + local line=$1 + local label=$2 + echo "${line#*"$label"=}" | cut -d'"' -f2 +} + +extract_value() { + local line=$1 + echo "${line##* }" +} + +#------------------------------------------------------------------------------- +# Fetch metrics +#------------------------------------------------------------------------------- +metrics_file=$(mktemp) +trap 'rm -f $metrics_file' EXIT + +curl -so "$metrics_file" "$METRICS_AT" || die "failed fetching from $METRICS_AT" + +#------------------------------------------------------------------------------- +# Parse metrics +#------------------------------------------------------------------------------- +declare -A log_index # log ID -> log index +declare -A log_size # log ID -> log size +declare -A log_timestamp # log ID -> log timestamp + +need_restart=0 +num_unexpected_crt=0 + +while IFS= read -r line; do + [[ $line =~ ^# ]] && continue + + case "$line" in + silentct_log_index*) + key=$(extract_label "$line" "log_name") + value=$(to_integer "$(extract_value "$line")") + log_index["$key"]=$value + ;; + silentct_log_size*) + key=$(extract_label "$line" "log_name") + value=$(to_integer "$(extract_value "$line")") + log_size["$key"]=$value + ;; + silentct_log_timestamp*) + key=$(extract_label "$line" "log_name") + value=$(to_integer "$(extract_value "$line")") + log_timestamp["$key"]=$((value / 1000)) + ;; + silentct_need_restart*) + need_restart=$(extract_value "$line") + ;; + silentct_unexpected_certificate*) + num_unexpected_crt=$((num_unexpected_crt + 1)) + ;; + esac +done <"$metrics_file" + +#------------------------------------------------------------------------------- +# Output warnings +#------------------------------------------------------------------------------- +now=$(date +%s) + +for log_name in "${!log_size[@]}"; do + backlog=$((log_size[$log_name] - log_index[$log_name])) + elapsed=$((now - log_timestamp[$log_name])) + + ((backlog < ALERT_BACKLOG)) || notice "$log_name -- backlog is at $backlog" + ((elapsed < ALERT_FRESHNESS)) || notice "$log_name -- latest timestamp at $(date -d @"${log_timestamp[$log_name]}")" +done + +[[ $need_restart == 0 ]] || notice "silentct-mon needs to be restarted" +[[ $num_unexpected_crt == 0 ]] || notice "$num_unexpected_crt unexpected certificate(s)" diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 0000000..2e21f12 --- /dev/null +++ b/docs/design.md @@ -0,0 +1,116 @@ +# silentct + +This document introduces a silent Certificate Transparency monitor design. + +## Setting + +We consider a setting where one or more trusted systems request certificates for +a list of domains. The domains that a system request certificates for may +overlap with the domains of other systems. For example, there may be two +distinct systems that host and request certificates for `www.example.org`. +Other examples of "systems" that request certificates could include +`jitsi.example.org`, `etherpad.example.org` and `gitlab.example.org`. + +The threat we are worried about is certificate mis-issuance. Due to considering +a multi-system setting with overlapping domains, no single system can be aware +of all legitimately issued certificates for the domains that are being managed. + +A certificate is considered mis-issued if it contains: + + 1. at least one domain that any of the trusted systems manage _but without any + of the trusted systems requesting that certificate to be issued_, or + 2. at least one subdomain of the domains that any of the trusted systems + manage _unless that subdomain is explicitly specified as out of scope_. + +The cause of certificate mis-issuance can vary, ranging from BGP and DNS hijacks +to certificate authorities that are coerced, compromised, or actively malicious. + +## Goals and non-scope + +The goal is to detect certificate mis-issuance. It is however out of scope to +detect certificate mis-issuance that happened in the past. In other words, if +the design described herein is put into operation at time `T`, then any +certificate mis-issuance that happened before time `T` is out of scope. This is +an important constraint that makes it _a lot less costly_ to bootstrap the +monitor. For example, old certificate backlogs can simply be ignored. + +It is also out of scope to detect certificate mis-issuance that targets web +browsers without Certificate Transparency enforcement. This is because we +cannot get a concise view of all certificates without Certificate Transparency. + +To detect certificate mis-issuance, we want to construct a monitor that: + + 1. _is easy to self-host_, because you trust yourself or can then (more + easily) find someone you trust to do the monitoring on your behalf, and + 2. _is silent_, so that there is little or no noise unless certificate + mis-issuance is actually suspected. In other words, there should not be a + notification every time a legitimate certificate is issued or renewed. + +The "silent" property helps a lot for system administrators that manage more +than a few certificates. It also helps in the third-party monitoring setting, +as it would not be more noisy to subscribe to notifications from >1 monitor. + +## Assumptions + + - The attacker is unable to control two independent logs that count towards + the SCT checks in web browsers. So, we need not worry about split-views and + can just download the logs while verifying that they are locally consistent. + - The systems that request certificates start in good states but may be + compromised sometime in the future. Detection of certificate mis-issuance + is then out of scope for all domains that the compromised systems managed. + - A mis-issued certificate will only be used to target connections from a + fixed set of IP addresses. A party that can distinguish between + certificates that are legitimate and mis-issued will never be targeted. + - A domain owner notices alerts about suspected certificate mis-issuance. The + monitor that generates these alerts is trusted and never compromised. + +## Architecture + +A monitor downloads all certificates that are issued by certificate authorities +from Certificate Transparency logs. The exact logs to download is automatically +updated using a list that Google publishes in signed form. All historical +updates to the list of logs is stored locally in case any issues are suspected. + +(It is possible to get INFO output whenever logs are added and removed. The +default verbosity is however NOTICE, which aims to be as silent as possible.) + +To filter out certificates that are not relevant, the monitor is configured with +a list of domains to match on. Only matching certificates will be stored, which +means there are nearly no storage requirements to run this type of monitor. + +To get the "silent" property, the monitor pulls the trusted systems for +legitimately issued certificates via HTTP GET. Alternatively, the monitor can +read a local file in case it is co-located with a single trusted system. The +monitor uses this as [feedback](./feedback.md) to filter the downloaded +certificates that matched. If a certificate is found that none of the trusted +systems made available, only then is an alert emitted (NOTICE level output). + +The communication channel between the trusted systems and the monitor can be +tampered with. For example, it may be plain HTTP or an HTTPS connection that +the attacker trivially hijacks by obtaining yet another mis-issued certificate. +Owning that the communication channel is insecure helps avoid misconfiguration. + +A shared secret is used for each system to authenticate with the monitor. This +secret is never shown on the wire: an HMAC key is derived from it, which is used +to produce message authentication codes. All a machine-in-the-middle attacker +can do is replay or block integrity-protected files that a system generated. + +"Replays" can happen either way because the monitor polls periodically, i.e., +the monitor needs to account for the fact that it may poll the same file twice. +Blocking can not be solved by cryptography and would simply result in alerts. + +## Related work + +The commercial version of `certspotter` supports a push-based method for +[authorizing][] legitimately issued certificates. The monitor does its +authentication using HTTP tokens. In contrast, the silentct design is: + + 1. Safe against attackers that MitM the communication to the monitor, i.e., + message authentication codes are used instead of HTTP access tokens. + 2. Applicable in asynchronous workflows, i.e., the monitor does not need to + always be online and listen for allowlist requests on a public address. + +The initial authors of silentct were not aware of Andrew Ayer's related work +before [this thread](https://follow.agwa.name/notice/AmyLDdYcAqF2p5sG24). + +[authorizing]: https://sslmate.com/help/reference/certspotter_authorization_api diff --git a/docs/feedback.md b/docs/feedback.md new file mode 100644 index 0000000..d79d57f --- /dev/null +++ b/docs/feedback.md @@ -0,0 +1,23 @@ +# Feedback + +This document describes the integrity-protected file format that a trusted +system uses when making legitimately issued certificates available to a monitor. + +## Format + + NAME MAC + <CERTIFICATE CHAIN> + ... + <CERTIFICATE CHAIN> + +`NAME`: identifier that the monitor uses to locate the shared secret. + +`MAC`: HMAC with SHA256 as the hash function, computed for line two and forward. +The shared HMAC key is derived as follows by the trusted system and the monitor: + + hkdf := hkdf.New(sha256.New, SECRET, []byte("silentct"), NAME) + key := make([]byte, 16) + io.ReadFull(hkdf, key) + +`<CERTIFICATE CHAIN>`: certificate chain in PEM format that the trusted system +considers legitimate. Can be repeated, then delimited by "silentct:separator". diff --git a/docs/help2man/reporting-bugs.help2man b/docs/help2man/reporting-bugs.help2man index 81a4147..cfe3036 100644 --- a/docs/help2man/reporting-bugs.help2man +++ b/docs/help2man/reporting-bugs.help2man @@ -1,12 +1,14 @@ [REPORTING BUGS] Use .B https://git.glasklar.is/rgdd/silentct/-/issues -for filing issues. -.br -Reach out to +for filing issues. To file issues without a GitLab account, email +.B rgdd-silentct-issues@incoming.glasklar.is +and wait for a maintainer to make the issue public. +You can also reach out to .B rgdd in room .B #certificate-transparency at .B OFTC.net -and Matrix. +and +.B matrix.org. diff --git a/docs/introduction.md b/docs/introduction.md deleted file mode 100644 index 0aab2cc..0000000 --- a/docs/introduction.md +++ /dev/null @@ -1,103 +0,0 @@ -# Silent Certificate Transparency - -This document introduces a silent Certificate Transparency monitor design. - -## Setting - -We consider a setting where one or more trusted _nodes_ request certificates for -a specified list of domain names. The domain names that a node requests -certificates for may overlap with the domain names of other nodes. For example, -there may be two distinct nodes that request certificates for a given domain. - -The threat we are worried about is certificate mis-issuance. Due to considering -a multi-node setting with overlapping domain names, no single node can be aware -of all legitimately issued certificates for the domain names that it manages. - -A certificate is considered mis-issued if it contains: - - 1. at least one domain name that any of the trusted nodes manage _but without - any of the trusted nodes requesting that certificate to be issued_, or - 2. at least one subdomain of the domain names that any of the trusted nodes - manage _unless that subdomain is explicitly specified as out of scope_. - -The cause of certificate mis-issuance can vary, ranging from BGP and DNS hijacks -to certificate authorities that are coerced, compromised, or actively malicious. - -## Goals and non-scope - -The goal is to detect certificate mis-issuance, not to prevent it. It is out of -scope to detect certificate mis-issuance that happened in the past. In other -words, if the architecture described herein is put into operation at time `T`, -then any certificate mis-issuance that happened before time `T` is out of scope. - -It is also out of scope to detect certificate mis-issuance that targets web -browsers without Certificate Transparency enforcement. This is because we -cannot get a concise view of all certificates without Certificate Transparency. - -To achieve the goal of certificate mis-issuance, we want a _monitor_ that: - - 1. _is easy to self-host_, because you trust yourself or can then find someone - else that is appropriate and willing to host your infrastructure, and - 2. _is silent_, so that there is little or no noise unless certificate - mis-issuance is suspected or other noteworthy log events are happening. - -## Assumptions - - - The attacker is unable to control two independent logs that count towards - the SCT checks in web browsers. So, we need not worry about split-views and - can just download the logs while verifying that they are locally consistent. - - The nodes that request certificates start in good states but may be - compromised sometime in the future. Detection of certificate mis-issuance - is then out of scope for all domains that the compromised nodes managed. - - A mis-issued certificate will only be used to target connections from a - fixed set of IP addresses. Any party that can distinguish between - certificates that are legitimate and mis-issued will never be targeted. - - A domain owner notices alerts about suspected certificate mis-issuance. The - monitor that generates these alerts is trusted and never compromised. - -## Architecture - -A monitor downloads all certificates that are issued by certificate authorities -from Certificate Transparency logs. The exact logs to download is automatically -updated using a list that Google publishes in signed form. All historical -updates to the list of logs is stored locally in case any issues are suspected. - -(It is possible to get INFO output whenever logs are added and removed. The -default verbosity is however NOTICE, which aims to be as silent as possible.) - -To filter out certificates that are not relevant, the monitor is configured with -a list of domains to match on. Only matching certificates will be stored, which -means there are nearly no storage requirements to run this type of monitor. - -To get the property of _silence_, the monitor pulls the trusted nodes via HTTP -GET for legitimately issued certificates (periodic job). The monitor will use -this feedback to filter the downloaded certificates that matched. If any -certificates are found that no node pushed to the monitor, an alert is printed. - -The communication channel between the trusted nodes and the monitor can be -tampered with. For example, it may be plain HTTP or an HTTPS connection that -the attacker trivially hijacks by obtaining yet another mis-issued certificate. -Owning that the communication channel is insecure helps avoid misconfiguration. - -A shared secret is used for each node to authenticate with the monitor. This -secret is never shown on the wire: an HMAC key is derived from it, which is used -to produce message authentication codes. All a machine-in-the-middle attacker -can do is replay or block integrity-protected submissions that a node generated. - -"Replays" can happen either way because the monitor polls periodically, i.e., -the monitor needs to account for the fact that it may poll the same thing twice. -Blocking can not be solved by cryptography and would simply result in alerts. - -## Further reading - -docdoc - -## Future ideas - - - Reduce the amount of bandwidth that the monitor spends downloading - certificates that are either way discarded (non-matches). This can be - achieved by introducing a _verifiable proxy_ supporting wildcard - (non-)membership proofs, see [verifiable light-weight monitoring][]. Ignore - the parts about changing the logs; that is easily solved by the proxy alone. - -[verifiable light-weight monitoring]: https://arxiv.org/pdf/1711.03952.pdf diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 0000000..627776a --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,96 @@ +# Metrics + +`silentct-mon` can output Prometheus metrics -- enable using the `-m` option. + +## Examples of useful alerts + + - **The monitor is falling behind on downloading a particular log**, e.g., + `silentct_log_size - silentct_log_index > 65536`. + - **The monitor hasn't seen a fresh timestamp from a particular log**, e.g., + `time() - silentct_log_timestamp > 24*60*60`. + - **The monitor needs restarting**, e.g., `silentct_need_restart != 0` + - **Unexpected certificates have been found**, e.g., + `silentct_unexpected_certificate_count > 0`. + +## `"silentct_error_counter"` + +``` +# HELP silentct_error_counter The number of errors propagated to the main loop. +# TYPE silentct_error_counter counter +silentct_error_counter 0 +``` + +Do not use for alerting, this metric is too noisy and currently used for debug. + +## `"silentct_log_index"` + +``` +# HELP silentct_log_index The next log entry to be downloaded. +# TYPE silentct_log_index gauge +silentct_log_index{log_id="4e75a3275c9a10c3385b6cd4df3f52eb1df0e08e1b8d69c0b1fa64b1629a39df",log_name="Google 'Argon2025h1'} 7.30980064e+08 +``` + +`log_id` is a unique log identifier in hex, computed as in RFC 6962 §3.2. + +`log_name` is a human-meaningful name of the log. + +## `"silentct_log_size"` + +``` +# HELP silentct_log_size The number of entries in the log. +# TYPE silentct_log_size gauge +silentct_log_size{log_id="4e75a3275c9a10c3385b6cd4df3f52eb1df0e08e1b8d69c0b1fa64b1629a39df",log_name="Google 'Argon2025h1'} 7.31044085e+08 +``` + +`log_id` is a unique log identifier in hex, computed as in RFC 6962 §3.2. + +`log_name` is a human-meaningful name of the log. + +## `"silentct_log_timestamp"` + +``` +# HELP silentct_log_timestamp The log's UNIX timestamp in ms. +# TYPE silentct_log_timestamp gauge +silentct_log_timestamp{log_id="4e75a3275c9a10c3385b6cd4df3f52eb1df0e08e1b8d69c0b1fa64b1629a39df",log_name="Google 'Argon2025h1'} 1.737202578179e+12 +``` + +`log_id` is a unique log identifier in hex, computed as in RFC 6962 §3.2. + +`log_name` is a human-meaningful name of the log. + +## `"silentct_need_restart"` + +``` +# HELP silentct_need_restart A non-zero value if the monitor needs restarting. +# TYPE silentct_need_restart gauge +silentct_need_restart 0 +``` + +Restarts are normally not needed; but here's a metric until the `silentct-mon` +implementation can assure that all corner-cases are handled without restarts. + +## `"silentct_unexpected_certificate_count"` + +``` +# HELP silentct_unexpected_certificate_count Number of certificates without any allowlisting +# TYPE silentct_unexpected_certificate_count gauge +silentct_unexpected_certificate_count{crt_sans="example.org www.example.org",log_id="4e75a3275c9a10c3385b6cd4df3f52eb1df0e08e1b8d69c0b1fa64b1629a39df",log_index="1234",log_name="Google 'Argon2025h1'} 1 +``` + +`crt_sans` are the subject alternative names in the unexpected certificate, +space separated. + +`log_id` is a unique log identifier in hex, computed as in RFC 6962 §3.2. + +`log_index` specifies the log entry that contains the unexpected certificate. + +`log_name` is a human-meaningful name of the log. + +See `STATE_DIRECTORY/crt_found/<log_id>-<log_index>.*` for further details. The +`.json` file contains the downloaded log entry. The `.ascii` file contains the +parsed leaf certificate in a human-readable format to make debugging easier. + +Allowlist an unexpected certificate by ingesting it from a trusted certificate +requester. Alternatively: stop the monitor, manually move the unexpected +certificate from the "alerting" dictionary to the "legitimate" dictionary in +`STATE_DIRECTORY/crt_index.json`, save, and then start the monitor again. diff --git a/docs/storage.md b/docs/storage.md deleted file mode 100644 index a0616ed..0000000 --- a/docs/storage.md +++ /dev/null @@ -1,3 +0,0 @@ -# Storage - -docdoc diff --git a/docs/submission.md b/docs/submission.md deleted file mode 100644 index 1d9c189..0000000 --- a/docs/submission.md +++ /dev/null @@ -1,22 +0,0 @@ -# Submission - -docdoc - -## Format - - NAME MAC - <PEM CHAIN> - silentct:separator - ... - <PEM CHAIN> - -`NAME`: identifier that the monitor uses to locate the right secret. - -`MAC`: HMAC with SHA256 as the hash function, computed for line two and forward. -The HMAC key is derived by the node and the monitor from their shared secret: - - hkdf := hkdf.New(sha256.New, SECRET, []byte("silentct"), NAME) - key := make([]byte, 16) - io.ReadFull(hkdf, key) - -`<PEM CHAIN>`: certificate chain in PEM format the node considers legitimate. @@ -1,18 +1,29 @@ module rgdd.se/silentct -go 1.19 +go 1.22.7 require ( - github.com/go-logr/logr v1.2.4 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/certificate-transparency-go v1.1.7 // indirect - github.com/google/trillian v1.5.3 // indirect - github.com/transparency-dev/merkle v0.0.2 // indirect - gitlab.torproject.org/rgdd/ct v0.0.0-20240101071140-1b7c55dcd5ba // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/net v0.17.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/grpc v1.59.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect - k8s.io/klog/v2 v2.100.1 // indirect + github.com/google/certificate-transparency-go v1.3.0 + github.com/google/trillian v1.7.0 + github.com/prometheus/client_golang v1.20.5 + github.com/transparency-dev/merkle v0.0.2 + gitlab.torproject.org/rgdd/ct v0.0.0 + golang.org/x/crypto v0.31.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + golang.org/x/net v0.31.0 // indirect + golang.org/x/sys v0.28.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241113202542-65e8d215514f // indirect + google.golang.org/grpc v1.68.0 // indirect + google.golang.org/protobuf v1.35.2 // indirect + k8s.io/klog/v2 v2.130.1 // indirect ) @@ -1,33 +1,48 @@ -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/certificate-transparency-go v1.1.7 h1:IASD+NtgSTJLPdzkthwvAG1ZVbF2WtFg4IvoA68XGSw= -github.com/google/certificate-transparency-go v1.1.7/go.mod h1:FSSBo8fyMVgqptbfF6j5p/XNdgQftAhSmXcIxV9iphE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/trillian v1.5.3 h1:3ioA5p09qz+U9/t2riklZtaQdZclaStp0/eQNfewNRg= -github.com/google/trillian v1.5.3/go.mod h1:p4tcg7eBr7aT6DxrAoILpc3uXNfcuAvZSnQKonVg+Eo= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/certificate-transparency-go v1.3.0 h1:+UhSNQAyA38Ed4CGfwOZeG4sJ030ELQZE4xtMFOxA7U= +github.com/google/certificate-transparency-go v1.3.0/go.mod h1:/xVlT13jyrOuJOXTW5PjCBCrHBtXUq/jT5UeW40xliQ= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/trillian v1.7.0 h1:Oib7mKRvZ0Z3GjvNcn2C4clRmFouEOkBcbzw7q8JlFI= +github.com/google/trillian v1.7.0/go.mod h1:JMp1zzzHe7j2m9m8P/eTWOaoon3R/SwgqUnFMhm4vfw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= github.com/transparency-dev/merkle v0.0.2/go.mod h1:pqSy+OXefQ1EDUVmAJ8MUhHB9TXGuzVAT58PqBoHz1A= -gitlab.torproject.org/rgdd/ct v0.0.0-20230508072727-1d1808eac7db h1:wR2Fc+fRDBkpabB1og472+HCoNtE4TJEuAHbSG4EIEs= -gitlab.torproject.org/rgdd/ct v0.0.0-20230508072727-1d1808eac7db/go.mod h1:dkEqBVulcsefxw3k5CX53bw88KUkTYcTRSJhMlj1Veg= -gitlab.torproject.org/rgdd/ct v0.0.0-20240101071140-1b7c55dcd5ba h1:ZoJTzvVdgbdejL6Ph2yJp/fkA/A6qJ+m0I8Xv6MDxFs= -gitlab.torproject.org/rgdd/ct v0.0.0-20240101071140-1b7c55dcd5ba/go.mod h1:dkEqBVulcsefxw3k5CX53bw88KUkTYcTRSJhMlj1Veg= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +gitlab.torproject.org/rgdd/ct v0.0.0 h1:YeVjFD14bFMMY+oIT6oGuG+8MzcOkFVcKgACqK1IbD0= +gitlab.torproject.org/rgdd/ct v0.0.0/go.mod h1:dkEqBVulcsefxw3k5CX53bw88KUkTYcTRSJhMlj1Veg= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241113202542-65e8d215514f h1:C1QccEa9kUwvMgEUORqQD9S17QesQijxjZ84sO82mfo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241113202542-65e8d215514f/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= +google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0= +google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= diff --git a/integration/smoke-test b/integration/smoke-test new file mode 100755 index 0000000..dacecd0 --- /dev/null +++ b/integration/smoke-test @@ -0,0 +1,100 @@ +#!/bin/bash + +set -eu + +function die() { + echo "ERROR: $*" >&2 + exit 1 +} + +function pass() { + echo "PASS: $*" >&2 +} + +function config() { + domain=$1; shift + remove_keys=$1; shift + + cat << EOF +{ + "monitor": [ + { + "bootstrap_at": "2025-01-01T00:00:00Z", + "suffix": "$domain" + } + ], + "remove_logs": [ +EOF + + mapfile -t keys <<< "$remove_keys" + for ((i = 0; i < ${#keys[@]}; i++)); do + key=${keys[i]} + if ((i == ${#keys[@]} - 1)); then + echo " \"$key\"" + else + echo " \"$key\"," + fi + done + + cat << EOF + ] +} +EOF +} + +#------------------------------------------------------------------------------- +# Options +#------------------------------------------------------------------------------- +TARGET_LOG=${TARGET_LOG:-"Google 'Argon2025h1' log"} # which log to test with +WAIT=${WAIT:-3} # seconds to wait for entries + +#------------------------------------------------------------------------------- +# Working directories +#------------------------------------------------------------------------------- +cd "$(dirname "$0")" + +dir=./testonly +rm -rf "$dir" +mkdir "$dir" + +#------------------------------------------------------------------------------- +# Metadata to configure monitoring of a single log +#------------------------------------------------------------------------------- +go run gitlab.torproject.org/rgdd/ct/cmd/ct-metadata@latest get source -n google > "$dir/metadata.json" +log_id=$(jq -r --arg desc "$TARGET_LOG" '.operators[].logs[] | select(.description == $desc) | .log_id' "$dir/metadata.json" | base64 -d | base16) +remove_keys=$(jq -r --arg desc "$TARGET_LOG" '.operators[].logs[] | select(.description != $desc) | .key' "$dir/metadata.json") + +config "www.example.org" "$remove_keys" > $dir/config.json +go run ../cmd/silentct-mon --bootstrap -c "$dir/config.json" -d "$dir/state" -C "dev:silentct" -v DEBUG + +pass "bootstrap the monitor with a single log" +sleep "$WAIT" + +#------------------------------------------------------------------------------- +# Figure out which domain name will be in the next log entry +#------------------------------------------------------------------------------- +next_index=$(jq '.next_index' "$dir/state/monitor_state/$log_id.json") +next_domain=$(go run github.com/google/certificate-transparency-go/client/ctclient@latest \ + get-entries --first "$next_index" --last "$next_index" --text --log_name "$TARGET_LOG" | \ + grep DNS | cut -d',' -f1 | cut -d':' -f2) + +[[ -n $next_domain ]] || die "failed to extract next domain name" + +#------------------------------------------------------------------------------- +# Run and find a match which turns into a certificate mis-issuance notice +#------------------------------------------------------------------------------- +config "$next_domain" "$remove_keys" >"$dir/config.json" +timeout 10s go run ../cmd/silentct-mon -c "$dir/config.json" -d "$dir/state" -C "dev:silentct" -p 1s -v DEBUG 2>&1 | tee "$dir/output.txt" +grep -q -E '([1-9][0-9]*|[1-9] matches)' "$dir/output.txt" || die "expected at least one match" +grep -q -F '[NOTICE] unexpected certificate:' "$dir/output.txt" || die "expected notice about unexpected certificate" + +pass "run the monitor and be warned of an unreported certificate" + +#------------------------------------------------------------------------------- +# Success +#------------------------------------------------------------------------------- +echo "---" >&2 +echo "All smoke tests passed" >&2 +echo "For interactive tests:" >&2 +echo "go run ../cmd/silentct-mon -c "$dir/config.json" -d "$dir/state" -C "dev:silentct" -p 15s -m localhost:8080 -v DEBUG" >&2 +echo "ALERT_BACKLOG=0 ALERT_FRESHNESS=0 ../contrib/silentct-check" >&2 diff --git a/internal/manager/manager.go b/internal/manager/manager.go index 6781d57..b839502 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -5,20 +5,24 @@ import ( "errors" "fmt" "os" + "strings" "time" "gitlab.torproject.org/rgdd/ct/pkg/metadata" "rgdd.se/silentct/internal/feedback" "rgdd.se/silentct/internal/logger" + "rgdd.se/silentct/internal/metrics" "rgdd.se/silentct/internal/monitor" "rgdd.se/silentct/pkg/policy" "rgdd.se/silentct/pkg/storage" + "rgdd.se/silentct/pkg/storage/loglist" ) type Config struct { Policy policy.Policy Bootstrap bool // Whether a new storage should be initialized from scratch Directory string // Path to a directory where everything will be stored + Metrics *metrics.Metrics // Optional Logger *logger.Logger // Where to output messages and with what verbosity @@ -82,6 +86,12 @@ func (mgr *Manager) Run(ctx context.Context) error { metadataTicker := time.NewTicker(mgr.MetadataRefreshInterval) defer metadataTicker.Stop() + alertTicker := time.NewTicker(mgr.AlertDelay) + defer alertTicker.Stop() + if err := mgr.alertJob(); err != nil { + return fmt.Errorf("unable to run alert job: %v\n", err) + } + shutdown := false for { select { @@ -89,6 +99,10 @@ func (mgr *Manager) Run(ctx context.Context) error { if err := mgr.metadataJob(ctx); err != nil { return fmt.Errorf("unable to run metadata job: %v", err) } + case <-alertTicker.C: + if err := mgr.alertJob(); err != nil { + return fmt.Errorf("unable to run alert job: %v\n", err) + } case ev := <-mgr.meventCh: if err := mgr.monitorJob(ev); err != nil { return fmt.Errorf("unable to run monitor job: %v", err) @@ -133,6 +147,7 @@ func (mgr *Manager) startupConfig() error { return err } mgr.mconfigCh <- monitor.MonitoredLog{Config: log, State: state} + mgr.Metrics.LogState(loglist.FormatLogName(log), state) } return nil } @@ -157,6 +172,7 @@ func (mgr *Manager) removeLogs(logs []metadata.Log) { state, _ := mgr.GetMonitorState(log) mgr.Logger.Infof("removing log %s with %d entries in its backlog\n", log.URL, state.TreeSize-state.NextIndex) mgr.mconfigCh <- monitor.MonitoredLog{Config: log} + mgr.Metrics.RemoveLogState(loglist.FormatLogName(log), state) } } @@ -168,10 +184,12 @@ func (mgr *Manager) addLogs(ctx context.Context, logs []metadata.Log) { mgr.Logger.Infof("adding log %s with existing state on disk\n", log.URL) } else if err != nil { mgr.Logger.Noticef("restart required: failed to bootstrap new log %s: %v\n", log.URL, err) + mgr.Metrics.NeedRestart() } else { mgr.Logger.Infof("bootstrapping log %s at next index 0\n", log.URL) } mgr.mconfigCh <- monitor.MonitoredLog{Config: log, State: state} + mgr.Metrics.LogState(loglist.FormatLogName(log), state) } } @@ -190,21 +208,47 @@ func (mgr *Manager) monitorJob(msg monitor.Event) error { if err := mgr.AddEntries(msg.State.LogID, msg.Matches); err != nil { return err } - return mgr.SetMonitorState(msg.State.LogID, msg.State) + if err := mgr.SetMonitorState(msg.State.LogID, msg.State); err != nil { + return err + } + for _, err := range msg.Errors { + mgr.errorJob(err) + } + + // no metrics update if the log has just been removed (final event) + name, err := mgr.Storage.LogList.LogName(msg.State.SignedTreeHead.LogID) + if err == nil { + mgr.Metrics.LogState(name, msg.State) + } + return nil } func (mgr *Manager) alertJob() error { + // See if there are any new unexpected certificates alerts, err := mgr.Index.TriggerAlerts() if err != nil { return err } for _, alert := range alerts { - mgr.Logger.Noticef("certificate mis-issuance? No node submitted certificate %s\n", alert.StoredAt) + mgr.Logger.Noticef("unexpected certificate: no allowlisting for crt_sans=\"%s\", see log_id=\"%x\" log_index=\"%d\"\n", strings.Join(alert.SANs, " "), alert.LogID, alert.LogIndex) + } + + // Update metrics for the current unexpected certificates + alerting := mgr.Storage.Index.Alerting() + var names []string + for _, alert := range alerting { + name, err := mgr.Storage.LogList.LogName(alert.LogID) + if err != nil { + name = "historic log" + } + names = append(names, name) } + mgr.Metrics.UnexpectedCertificateCount(names, mgr.Storage.Index.Alerting()) return nil } func (mgr *Manager) errorJob(err error) error { mgr.Logger.Debugf("received error: %v\n", err) + mgr.Metrics.CountError() return nil } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..aae46cd --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,114 @@ +package metrics + +import ( + "fmt" + "strings" + + "github.com/prometheus/client_golang/prometheus" + "rgdd.se/silentct/internal/monitor" + "rgdd.se/silentct/pkg/storage/index" +) + +type Metrics struct { + errorCounter prometheus.Counter + logIndex *prometheus.GaugeVec + logSize *prometheus.GaugeVec + logTimestamp *prometheus.GaugeVec + needRestart prometheus.Gauge + unexpectedCertificateCount *prometheus.GaugeVec +} + +func NewMetrics(registry *prometheus.Registry) *Metrics { + m := &Metrics{ + errorCounter: prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "silentct_error_counter", + Help: "The number of errors propagated to the main loop.", + }, + ), + logIndex: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "silentct_log_index", + Help: "The next log entry to be downloaded.", + }, + []string{"log_id", "log_name"}, + ), + logSize: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "silentct_log_size", + Help: "The number of entries in the log.", + }, + []string{"log_id", "log_name"}, + ), + logTimestamp: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "silentct_log_timestamp", + Help: "The log's UNIX timestamp in ms.", + }, + []string{"log_id", "log_name"}, + ), + needRestart: prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: "silentct_need_restart", + Help: "A non-zero value if the monitor needs restarting.", + }, + ), + unexpectedCertificateCount: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "silentct_unexpected_certificate_count", + Help: "Number of certificates without any allowlisting", + }, + []string{"log_id", "log_name", "log_index", "crt_sans"}, + ), + } + registry.MustRegister( + m.errorCounter, + m.logIndex, + m.logSize, + m.logTimestamp, + m.needRestart, + m.unexpectedCertificateCount, + ) + return m +} + +func (m *Metrics) LogState(logName string, state monitor.State) { + labels := prometheus.Labels{ + "log_id": fmt.Sprintf("%x", state.LogID[:]), + "log_name": logName, + } + m.logIndex.With(labels).Set(float64(state.NextIndex)) + m.logSize.With(labels).Set(float64(state.TreeSize)) + m.logTimestamp.With(labels).Set(float64(state.Timestamp)) +} + +func (m *Metrics) RemoveLogState(logName string, state monitor.State) { + labels := prometheus.Labels{ + "log_id": fmt.Sprintf("%x", state.LogID[:]), + "log_name": logName, + } + m.logIndex.Delete(labels) + m.logSize.Delete(labels) + m.logTimestamp.Delete(labels) +} + +func (m *Metrics) UnexpectedCertificateCount(logNames []string, alerts []index.CertificateInfo) { + m.unexpectedCertificateCount.Reset() + for i, alert := range alerts { + labels := prometheus.Labels{ + "crt_sans": strings.Join(alert.SANs, " "), + "log_id": fmt.Sprintf("%x", alert.LogID), + "log_name": logNames[i], + "log_index": fmt.Sprintf("%d", alert.LogIndex), + } + m.unexpectedCertificateCount.With(labels).Set(1) + } +} + +func (m *Metrics) CountError() { + m.errorCounter.Inc() +} + +func (m *Metrics) NeedRestart() { + m.needRestart.Set(float64(1)) +} diff --git a/internal/monitor/backoff.go b/internal/monitor/backoff.go new file mode 100644 index 0000000..63c5f55 --- /dev/null +++ b/internal/monitor/backoff.go @@ -0,0 +1,56 @@ +package monitor + +import ( + "context" + + ct "github.com/google/certificate-transparency-go" + "github.com/google/certificate-transparency-go/client" + "github.com/google/certificate-transparency-go/jsonclient" + "github.com/google/trillian/client/backoff" +) + +// backoffClient wraps client.LogClient so that we always backoff on get-entries +// 4XX and 5XX. Backoff is on by default for get-sth already, and our silentct +// usage is guaranteed to not do any hammering on any of the proof endpoints. +// +// For reference on this issue, see: +// https://github.com/google/certificate-transparency-go/issues/898 +type backoffClient struct { + cli *client.LogClient +} + +func (bc *backoffClient) BaseURI() string { + return bc.cli.BaseURI() +} + +func (bc *backoffClient) GetSTH(ctx context.Context) (*ct.SignedTreeHead, error) { + return bc.cli.GetSTH(ctx) +} + +func (bc *backoffClient) GetSTHConsistency(ctx context.Context, first, second uint64) ([][]byte, error) { + return bc.cli.GetSTHConsistency(ctx, first, second) +} + +func (bc *backoffClient) GetProofByHash(ctx context.Context, hash []byte, treeSize uint64) (*ct.GetProofByHashResponse, error) { + return bc.cli.GetProofByHash(ctx, hash, treeSize) +} + +func (bc *backoffClient) GetRawEntries(ctx context.Context, start, end int64) (*ct.GetEntriesResponse, error) { + rsp, err := bc.cli.GetRawEntries(ctx, start, end) + if err != nil { + jcErr, ok := err.(jsonclient.RspError) + if !ok { + return rsp, err + } + if jcErr.StatusCode < 400 || jcErr.StatusCode >= 600 { + return rsp, err + } + // This ensures we never start hammering when the status code is 4XX or + // 5XX. Probably not the right thing to do in all cases, but since the + // download library we're using starts hammering if the log suddenly + // serves something unexpected this seems like a good safety precaution. + // Users of the silentct monitor eventually notice they get no entries. + return rsp, backoff.RetriableErrorf("get-entries: %v", err) + } + return rsp, err +} diff --git a/internal/monitor/monitor.go b/internal/monitor/monitor.go index ffe7f75..2575977 100644 --- a/internal/monitor/monitor.go +++ b/internal/monitor/monitor.go @@ -18,6 +18,7 @@ import ( "net/http" "os" "sync" + "time" ct "github.com/google/certificate-transparency-go" "github.com/google/certificate-transparency-go/client" @@ -68,6 +69,8 @@ type Config struct { Logger *logger.Logger // Debug prints only (no output by default) Contact string // Something that help log operators get in touch ChunkSize uint // Min number of leaves to propagate a chunk without matches + ChunkTime time.Duration // But always send chunks (if there are any) with this interval + ExitTime time.Duration // Maximum amount of time to spend on a graceful exit BatchSize uint // Max number of certificates to accept per worker NumWorkers uint // Number of parallel workers to use for each log } @@ -94,6 +97,12 @@ func New(cfg Config, evCh chan Event, cfgCh chan MonitoredLog, errCh chan error) if cfg.ChunkSize == 0 { cfg.ChunkSize = 4096 } + if cfg.ChunkTime == 0 { + cfg.ChunkTime = 10 * time.Minute + } + if cfg.ExitTime == 0 { + cfg.ExitTime = 10 * time.Second + } if cfg.BatchSize == 0 { cfg.BatchSize = 1024 } @@ -164,7 +173,8 @@ func (mon *Monitor) newTailRFC6962(log MonitoredLog) (tail, error) { return tail{}, err } - return tail{cfg: mon.cfg, scanner: cli, checker: cli, matcher: mon.matcher}, nil + bc := &backoffClient{cli: cli} + return tail{cfg: mon.cfg, scanner: bc, checker: bc, matcher: mon.matcher}, nil } func (mon *Monitor) newTailTile(cfg MonitoredLog) (tail, error) { diff --git a/internal/monitor/tail.go b/internal/monitor/tail.go index d00ebe6..2603e81 100644 --- a/internal/monitor/tail.go +++ b/internal/monitor/tail.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sync" + "time" "github.com/google/certificate-transparency-go/client" "github.com/google/certificate-transparency-go/scanner" @@ -75,31 +76,70 @@ func (t *tail) run(ctx context.Context, mon MonitoredLog, eventCh chan Event, er } func (t *tail) sequence(ctx context.Context, mon MonitoredLog, eventCh chan Event, errorCh chan error, chunkCh chan *chunk) { + var failedAt time.Time state := mon.State heap := newChunks() + sendChunk := func(ctx context.Context, force bool) { + if !failedAt.IsZero() && failedAt.Add(30*time.Second).After(time.Now()) { + return // ensures we don't spam get-sth and proof endpoints + } + + if heap.gap(state.NextIndex) { + return // nothing to send yet + } + c := heap.pop() + if !force && len(c.matches) == 0 && len(c.leafHashes) < int(t.cfg.ChunkSize) { + heap.push(c) + return // wait for a larger chunk before batch verification + } + + nextState, err := t.nextState(ctx, state, c) + if err != nil { + failedAt = time.Now() + errorCh <- err + heap.push(c) + return + } + + state = nextState + eventCh <- Event{State: state, Matches: c.matches, Errors: c.errors} + } + refreshSTH := func(ctx context.Context) { + timestamp := time.UnixMilli(int64(state.Timestamp)) + timestamp = timestamp.Add(t.cfg.ChunkTime) + timestamp = timestamp.Add(-1 * time.Second) + if timestamp.After(time.Now()) { + return + } + + // Looks like we haven't send any chunks the past ChunkTime time units. + // Get a newer tree head so the timestamp can be used for freshness. + nextState, err := t.nextConsistentState(ctx, state) + if err != nil { + errorCh <- err + return + } + state = nextState + eventCh <- Event{State: state} + } + + sendTicker := time.NewTicker(t.cfg.ChunkTime) + defer sendTicker.Stop() + for { select { case <-ctx.Done(): - return // FIXME: check if we can pop something before return + dctx, cancel := context.WithTimeout(context.Background(), t.cfg.ExitTime) + defer cancel() + sendChunk(dctx, true) + refreshSTH(dctx) + return + case <-sendTicker.C: + sendChunk(ctx, true) + refreshSTH(ctx) case c := <-chunkCh: heap.push(c) - if heap.gap(state.NextIndex) { - continue - } - c = heap.pop() - if len(c.matches) == 0 && len(c.leafHashes) < int(t.cfg.ChunkSize) { - heap.push(c) - continue // FIXME: don't trigger if we havn't run nextState for too long - } - nextState, err := t.nextState(ctx, state, c) - if err != nil { - errorCh <- err - heap.push(c) - continue - } - - state = nextState - eventCh <- Event{State: state, Matches: c.matches, Errors: c.errors} + sendChunk(ctx, false) } } } @@ -121,6 +161,9 @@ func (t *tail) nextConsistentState(ctx context.Context, state State) (State, err if err != nil { return State{}, fmt.Errorf("%s: get-sth: %v", t.checker.BaseURI(), err) } + if sth.Timestamp < state.Timestamp { + return State{}, fmt.Errorf("%s: get-sth: timestamp is shrinking", t.checker.BaseURI()) + } sth.LogID = state.SignedTreeHead.LogID oldSize := state.TreeSize oldRoot := state.SHA256RootHash diff --git a/pkg/policy/node.go b/pkg/policy/node.go index 607dbc3..d933e7f 100644 --- a/pkg/policy/node.go +++ b/pkg/policy/node.go @@ -11,10 +11,10 @@ import ( ) type Node struct { - Name string `json:"name"` // Artbirary node name to authenticate - Secret string `json:"secret"` // Arbitrary node secret for authentication - URL string `json:"url"` // Where the node's submissions can be downloaded - Domains []string `json:"issues"` // Exact-match domain names allowed to be issued + Name string `json:"name"` // Artbirary node name to authenticate + Secret string `json:"secret"` // Arbitrary node secret for authentication + URL string `json:"location"` // Where the node's submissions can be downloaded + Domains []string `json:"requests"` // Exact-match domain names allowed to be issued key [16]byte } diff --git a/pkg/policy/policy.go b/pkg/policy/policy.go index 8ee4867..3c2fec2 100644 --- a/pkg/policy/policy.go +++ b/pkg/policy/policy.go @@ -10,7 +10,7 @@ import ( type Policy struct { Monitor Wildcards `json:"monitor"` - Nodes []Node `json:"nodes"` + Nodes []Node `json:"certificate_requesters"` // Optional StaticLogs []metadata.Log `json:"static_logs"` diff --git a/pkg/policy/wildcard.go b/pkg/policy/wildcard.go index c67e1d9..b0bdf61 100644 --- a/pkg/policy/wildcard.go +++ b/pkg/policy/wildcard.go @@ -38,7 +38,7 @@ func (w *Wildcards) match(sans []string, notAfter time.Time) bool { // 2. the certificate expired before the BootstrapAt timestamp. type Wildcard struct { BootstrapAt time.Time `json:"bootstrap_at"` - Wildcard string `json:"wildcard"` + Wildcard string `json:"suffix"` Excludes []string `json:"excludes,omitempty"` } @@ -52,10 +52,10 @@ func (w *Wildcard) UnmarshalJSON(data []byte) error { func (w *Wildcard) Validate() error { if w.BootstrapAt.IsZero() { - return fmt.Errorf("bootstrap time is required") + return fmt.Errorf("bootstrap_at is required") } if len(w.Wildcard) == 0 { - return fmt.Errorf("wildcard is required") + return fmt.Errorf("suffix is required") } return nil } diff --git a/pkg/storage/index/index.go b/pkg/storage/index/index.go index 36a9334..c85a9e9 100644 --- a/pkg/storage/index/index.go +++ b/pkg/storage/index/index.go @@ -8,6 +8,7 @@ import ( "fmt" "time" + "github.com/google/certificate-transparency-go/x509util" "rgdd.se/silentct/internal/ioutil" "rgdd.se/silentct/internal/monitor" "rgdd.se/silentct/pkg/crtutil" @@ -55,7 +56,7 @@ func (ix *Index) AddChain(node string, pem []byte) error { var crtID CertificateID crtID.Set(chain[0]) path := fmt.Sprintf("%s/%s-%s.pem", ix.cfg.TrustDirectory, node, crtID) - if !ix.mem.addChain(crtID, path) { + if !ix.mem.addChain(path, crtID, chain[0].DNSNames) { return nil // duplicate } @@ -75,11 +76,14 @@ func (ix *Index) AddEntries(logID [sha256.Size]byte, entries []monitor.LogEntry) var crtID CertificateID crtID.Set(crt) path := fmt.Sprintf("%s/%x-%d.json", ix.cfg.MatchDirectory, logID[:], entry.LeafIndex) - if !ix.mem.addEntry(crtID, path) { + if !ix.mem.addEntry(path, crtID, crt.DNSNames, logID, entry.LeafIndex) { return nil // duplicate } - - return ioutil.CommitJSON(path, entry) + if err := ioutil.CommitJSON(path, entry); err != nil { + return err + } + path = fmt.Sprintf("%s/%x-%d.ascii", ix.cfg.MatchDirectory, logID[:], entry.LeafIndex) + return ioutil.CommitData(path, []byte(x509util.CertificateToString(&crt))) } for _, entry := range entries { @@ -98,6 +102,13 @@ func (ix *Index) TriggerAlerts() ([]CertificateInfo, error) { return alerts, ioutil.CommitJSON(ix.cfg.IndexFile, ix.mem) } +func (ix *Index) Alerting() (ret []CertificateInfo) { + for _, ci := range ix.mem.Alerting { + ret = append(ret, ci[0]) // one is enough for the same crt ID + } + return +} + func (index *Index) Validate() error { return nil // FIXME: check that the index is populated with valid values } diff --git a/pkg/storage/index/inmem.go b/pkg/storage/index/inmem.go index ba48bc1..6184cad 100644 --- a/pkg/storage/index/inmem.go +++ b/pkg/storage/index/inmem.go @@ -16,8 +16,12 @@ func (crtID *CertificateID) Set(crt x509.Certificate) { } type CertificateInfo struct { - ObservedAt time.Time `json:"observed_at"` - StoredAt string `json:"stored_at"` + ObservedAt time.Time `json:"observed_at"` + StoredAt string `json:"stored_at"` + SerialNumber CertificateID `json:"serial_number"` + SANs []string `json:"crt_sans"` + LogID [32]byte `json:"log_id,omitempty"` + LogIndex uint64 `json:"log_index,omitempty"` } // index is an in-memory index of certificates @@ -65,12 +69,12 @@ func (ix *index) triggerAlerts(delay time.Duration) []CertificateInfo { return alerts } -func (ix *index) addChain(crtID CertificateID, path string) bool { +func (ix *index) addChain(path string, crtID CertificateID, sans []string) bool { if _, ok := ix.Legitimate[crtID]; ok { return false // we already marked this certificate as "good" } - entry := CertificateInfo{ObservedAt: time.Now(), StoredAt: path} + entry := CertificateInfo{ObservedAt: time.Now(), StoredAt: path, SerialNumber: crtID, SANs: sans} crtInfos := []CertificateInfo{entry} if v, ok := ix.Alerting[crtID]; ok { crtInfos = append(crtInfos, v...) @@ -84,8 +88,8 @@ func (ix *index) addChain(crtID CertificateID, path string) bool { return true // index updated such that this certificate is marked as "good" } -func (ix *index) addEntry(crtID CertificateID, path string) bool { - crtInfo := CertificateInfo{ObservedAt: time.Now(), StoredAt: path} +func (ix *index) addEntry(path string, crtID CertificateID, sans []string, logID [32]byte, logIndex uint64) bool { + crtInfo := CertificateInfo{ObservedAt: time.Now(), StoredAt: path, SerialNumber: crtID, SANs: sans, LogID: logID, LogIndex: logIndex} if _, ok := ix.Legitimate[crtID]; ok { return add(ix.Legitimate, crtID, crtInfo) } else if _, ok := ix.Alerting[crtID]; ok { diff --git a/pkg/storage/loglist/loglist.go b/pkg/storage/loglist/loglist.go index a37cb32..f282113 100644 --- a/pkg/storage/loglist/loglist.go +++ b/pkg/storage/loglist/loglist.go @@ -72,6 +72,10 @@ func New(cfg Config) (LogList, error) { return ll, nil } +func (ll *LogList) LogName(logID [32]byte) (string, error) { + return metadataLogName(ll.md, logID) +} + func (ll *LogList) IsRecent() bool { return time.Now().Before(ll.md.CreatedAt.Add(ll.cfg.MetadataIsRecent)) } diff --git a/pkg/storage/loglist/metadata.go b/pkg/storage/loglist/metadata.go index adacf81..96d035c 100644 --- a/pkg/storage/loglist/metadata.go +++ b/pkg/storage/loglist/metadata.go @@ -1,6 +1,11 @@ package loglist -import "gitlab.torproject.org/rgdd/ct/pkg/metadata" +import ( + "fmt" + "strings" + + "gitlab.torproject.org/rgdd/ct/pkg/metadata" +) // FIXME: helpers that should probably be in the upstream package @@ -13,6 +18,25 @@ func metadataFindLog(md metadata.Metadata, target metadata.Log) bool { return false } +func metadataLogName(md metadata.Metadata, targetID [32]byte) (string, error) { + for _, operator := range md.Operators { + for _, log := range operator.Logs { + id, _ := log.Key.ID() + if id == targetID { + return FormatLogName(log), nil + } + } + } + return "", fmt.Errorf("no match for log ID: %x", targetID[:]) +} + +func FormatLogName(log metadata.Log) string { + if log.Description != nil { + return *log.Description + } + return strings.TrimSuffix("https://", string(log.URL)) +} + func findLog(logs []metadata.Log, target metadata.Log) bool { targetID, _ := target.Key.ID() for _, log := range logs { |